From 48c70b8f7dfd00a018abbac50ea987f54fa4db51 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 19 Dec 2024 11:08:17 +0900 Subject: [PATCH 01/33] gh-115999: Enable BINARY_SUBSCR_GETITEM for free-threaded build (gh-127737) --- Include/internal/pycore_opcode_metadata.h | 4 +-- Include/internal/pycore_typeobject.h | 1 + Include/internal/pycore_uop_metadata.h | 2 +- Lib/test/test_opcache.py | 19 +++++++++++- Objects/typeobject.c | 25 ++++++++++++++++ Programs/test_frozenmain.h | 2 +- Python/bytecodes.c | 22 +++++++------- Python/executor_cases.c.h | 32 ++++++++++++--------- Python/generated_cases.c.h | 19 ++++++------ Python/optimizer_bytecodes.c | 3 +- Python/optimizer_cases.c.h | 16 ++++++++--- Python/specialize.c | 35 ++++++++++++----------- 12 files changed, 118 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 28aa1120414337..d2ae8928a8fe8f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -994,7 +994,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case BINARY_SUBSCR: { - *effect = 0; + *effect = 1; return 0; } case BINARY_SUBSCR_DICT: { @@ -1002,7 +1002,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case BINARY_SUBSCR_GETITEM: { - *effect = 0; + *effect = 1; return 0; } case BINARY_SUBSCR_LIST_INT: { diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 7b39d07f976ee3..581153344a8e05 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -278,6 +278,7 @@ typedef int (*_py_validate_type)(PyTypeObject *); // and if the validation is passed, it will set the ``tp_version`` as valid // tp_version_tag from the ``ty``. extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index dd775d3f7d3cdd..eadfda472a7270 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -722,7 +722,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _BINARY_SUBSCR_CHECK_FUNC: return 0; case _BINARY_SUBSCR_INIT_CALL: - return 2; + return 3; case _LIST_APPEND: return 1; case _SET_ADD: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 0a7557adc4763b..94709e2022550a 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1069,7 +1069,7 @@ def write(items): opname = "STORE_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_unpack_sequence_list(self): def get_items(): items = [] @@ -1245,6 +1245,14 @@ def f(o, n): f(test_obj, 1) self.assertEqual(test_obj.b, 0) +# gh-127274: BINARY_SUBSCR_GETITEM will only cache __getitem__ methods that +# are deferred. We only defer functions defined at the top-level. +class CGetItem: + def __init__(self, val): + self.val = val + def __getitem__(self, item): + return self.val + class TestSpecializer(TestBase): @@ -1520,6 +1528,15 @@ def binary_subscr_str_int(): self.assert_specialized(binary_subscr_str_int, "BINARY_SUBSCR_STR_INT") self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") + def binary_subscr_getitems(): + items = [CGetItem(i) for i in range(100)] + for i in range(100): + self.assertEqual(items[i][i], i) + + binary_subscr_getitems() + self.assert_specialized(binary_subscr_getitems, "BINARY_SUBSCR_GETITEM") + self.assert_no_opcode(binary_subscr_getitems, "BINARY_SUBSCR") + if __name__ == "__main__": unittest.main() diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2068d6aa9be52b..7f95b519561e68 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5679,6 +5679,31 @@ _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, return can_cache; } +int +_PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version) +{ + if (!descriptor || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)ht)->tp_version_tag == tp_version; + // This pointer is invalidated by PyType_Modified (see the comment on + // struct _specialization_cache): + PyFunctionObject *func = (PyFunctionObject *)descriptor; + uint32_t version = _PyFunction_GetVersionForCurrentState(func); + can_cache = can_cache && _PyFunction_IsVersionValid(version); +#ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(descriptor); +#endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(ht->_spec_cache.getitem, descriptor); + FT_ATOMIC_STORE_UINT32_RELAXED(ht->_spec_cache.getitem_version, version); + } + END_TYPE_LOCK(); + return can_cache; +} + static void set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) { diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index c936622c020e3c..99b0fa48e01c8b 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,6 +1,6 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { - 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, + 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,168,0,0,0,149,0,89,0,79,0, 70,0,111,0,89,0,79,0,70,1,111,1,88,2,31,0, 79,1,49,1,0,0,0,0,0,0,29,0,88,2,31,0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 772b46d17ec198..b67264f0440869 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -865,26 +865,24 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_SUBSCR_CHECK_FUNC, (container, unused -- container, unused)) { + op(_BINARY_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - DEOPT_IF(getitem == NULL); - assert(PyFunction_Check(getitem)); - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(((PyFunctionObject *)getitem)->func_version != cached_version); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem); + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _PyInterpreterFrame* )) { - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); + op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _PyInterpreterFrame* )) { + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; INPUTS_DEAD(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 55e9c3aa2db64d..de61a64a6e3374 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1125,6 +1125,7 @@ case _BINARY_SUBSCR_CHECK_FUNC: { _PyStackRef container; + _PyStackRef getitem; container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { @@ -1132,42 +1133,45 @@ JUMP_TO_JUMP_TARGET(); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - if (getitem == NULL) { + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyFunction_Check(getitem)); - uint32_t cached_version = ht->_spec_cache.getitem_version; - if (((PyFunctionObject *)getitem)->func_version != cached_version) { + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); + stack_pointer[0] = getitem; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_INIT_CALL: { + _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; _PyInterpreterFrame *new_frame; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = 2 ; - stack_pointer[-2].bits = (uintptr_t)new_frame; - stack_pointer += -1; + stack_pointer[-3].bits = (uintptr_t)new_frame; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 94343f953221eb..8a89ba890fd9c9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -505,6 +505,7 @@ INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); _PyStackRef container; + _PyStackRef getitem; _PyStackRef sub; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -518,23 +519,21 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - DEOPT_IF(getitem == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(getitem)); - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(((PyFunctionObject *)getitem)->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem); + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } // _BINARY_SUBSCR_INIT_CALL { sub = stack_pointer[-1]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = 2 ; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 0b8aff02367e31..e60c0d38425bfe 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -349,9 +349,10 @@ dummy_func(void) { GETLOCAL(this_instr->operand0) = res; } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { (void)container; (void)sub; + (void)getitem; new_frame = NULL; ctx->done = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f4fbe8c8aa0480..33b34d6fa0d3f9 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -592,21 +592,29 @@ } case _BINARY_SUBSCR_CHECK_FUNC: { + _Py_UopsSymbol *getitem; + getitem = sym_new_not_null(ctx); + stack_pointer[0] = getitem; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_INIT_CALL: { + _Py_UopsSymbol *getitem; _Py_UopsSymbol *sub; _Py_UopsSymbol *container; _Py_UOpsAbstractFrame *new_frame; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; (void)container; (void)sub; + (void)getitem; new_frame = NULL; ctx->done = true; - stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; - stack_pointer += -1; + stack_pointer[-3] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index 6eb298217ec2d3..6c45320f95db8e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1096,6 +1096,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; @@ -1165,6 +1166,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (version == 0) { return -1; } + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; @@ -1781,12 +1783,12 @@ _Py_Specialize_BinarySubscr( specialized_op = BINARY_SUBSCR_DICT; goto success; } -#ifndef Py_GIL_DISABLED - PyTypeObject *cls = Py_TYPE(container); - PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); + unsigned int tp_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE); + Py_DECREF(descriptor); goto fail; } PyFunctionObject *func = (PyFunctionObject *)descriptor; @@ -1794,30 +1796,29 @@ _Py_Specialize_BinarySubscr( int kind = function_kind(fcode); if (kind != SIMPLE_FUNCTION) { SPECIALIZATION_FAIL(BINARY_SUBSCR, kind); + Py_DECREF(descriptor); goto fail; } if (fcode->co_argcount != 2) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + Py_DECREF(descriptor); goto fail; } - uint32_t version = _PyFunction_GetVersionForCurrentState(func); - if (!_PyFunction_IsVersionValid(version)) { - SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS); - goto fail; - } + + PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER); + Py_DECREF(descriptor); goto fail; } - PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; - // This pointer is invalidated by PyType_Modified (see the comment on - // struct _specialization_cache): - ht->_spec_cache.getitem = descriptor; - ht->_spec_cache.getitem_version = version; - specialized_op = BINARY_SUBSCR_GETITEM; - goto success; + if (_PyType_CacheGetItemForSpecialization(ht, descriptor, (uint32_t)tp_version)) { + specialized_op = BINARY_SUBSCR_GETITEM; + Py_DECREF(descriptor); + goto success; + } } -#endif // Py_GIL_DISABLED + Py_XDECREF(descriptor); SPECIALIZATION_FAIL(BINARY_SUBSCR, binary_subscr_fail_kind(container_type, sub)); fail: @@ -2617,6 +2618,7 @@ _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER); goto failure; @@ -2645,6 +2647,7 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER); goto failure; From 46dc1ba9c6e8b95635fa27607d01d6108d8f677e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Dec 2024 11:27:29 +0100 Subject: [PATCH 02/33] gh-128069: brew link --overwrite tcl-tk@8 to prevent conflict with GitHub image's version (#128090) brew link --overwrite tcl-tk@8 to prevent conflict with GitHub image's version Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 36ae3e27207e37..6fa389b2d66e5e 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -42,7 +42,7 @@ jobs: run: | brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make # Because alternate versions are not symlinked into place by default: - brew link tcl-tk@8 + brew link --overwrite tcl-tk@8 - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ From 3c168f7f79d1da2323d35dcf88c2d3c8730e5df6 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 19 Dec 2024 17:08:32 +0530 Subject: [PATCH 03/33] gh-128013: fix data race in `PyUnicode_AsUTF8AndSize` on free-threading (#128021) --- Lib/test/test_capi/test_unicode.py | 20 +++++++++++- Objects/unicodeobject.c | 49 +++++++++++++++++++----------- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 65d8242ad3fc60..3408c10f426058 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1,7 +1,7 @@ import unittest import sys from test import support -from test.support import import_helper +from test.support import threading_helper try: import _testcapi @@ -1005,6 +1005,24 @@ def test_asutf8(self): self.assertRaises(TypeError, unicode_asutf8, [], 0) # CRASHES unicode_asutf8(NULL, 0) + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @threading_helper.requires_working_threading() + def test_asutf8_race(self): + """Test that there's no race condition in PyUnicode_AsUTF8()""" + unicode_asutf8 = _testcapi.unicode_asutf8 + from threading import Thread + + data = "😊" + + def worker(): + for _ in range(1000): + self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0') + + threads = [Thread(target=worker) for _ in range(10)] + with threading_helper.start_threads(threads): + pass + + @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_asutf8andsize(self): diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 53be6f5b9017a3..1aab9cf37768a8 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -114,7 +114,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently static inline char* _PyUnicode_UTF8(PyObject *op) { - return (_PyCompactUnicodeObject_CAST(op)->utf8); + return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8); } static inline char* PyUnicode_UTF8(PyObject *op) @@ -130,7 +130,7 @@ static inline char* PyUnicode_UTF8(PyObject *op) static inline void PyUnicode_SET_UTF8(PyObject *op, char *utf8) { - _PyCompactUnicodeObject_CAST(op)->utf8 = utf8; + FT_ATOMIC_STORE_PTR_RELEASE(_PyCompactUnicodeObject_CAST(op)->utf8, utf8); } static inline Py_ssize_t PyUnicode_UTF8_LENGTH(PyObject *op) @@ -700,16 +700,17 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) CHECK(ascii->state.compact == 0); CHECK(data != NULL); if (ascii->state.ascii) { - CHECK(compact->utf8 == data); + CHECK(_PyUnicode_UTF8(op) == data); CHECK(compact->utf8_length == ascii->length); } else { - CHECK(compact->utf8 != data); + CHECK(_PyUnicode_UTF8(op) != data); } } - - if (compact->utf8 == NULL) +#ifndef Py_GIL_DISABLED + if (_PyUnicode_UTF8(op) == NULL) CHECK(compact->utf8_length == 0); +#endif } /* check that the best kind is used: O(n) operation */ @@ -1156,8 +1157,8 @@ resize_compact(PyObject *unicode, Py_ssize_t length) if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - PyUnicode_SET_UTF8(unicode, NULL); PyUnicode_SET_UTF8_LENGTH(unicode, 0); + PyUnicode_SET_UTF8(unicode, NULL); } #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); @@ -1210,8 +1211,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - PyUnicode_SET_UTF8(unicode, NULL); PyUnicode_SET_UTF8_LENGTH(unicode, 0); + PyUnicode_SET_UTF8(unicode, NULL); } data = (PyObject *)PyObject_Realloc(data, new_size); @@ -1221,8 +1222,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) } _PyUnicode_DATA_ANY(unicode) = data; if (share_utf8) { - PyUnicode_SET_UTF8(unicode, data); PyUnicode_SET_UTF8_LENGTH(unicode, length); + PyUnicode_SET_UTF8(unicode, data); } _PyUnicode_LENGTH(unicode) = length; PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0); @@ -4216,6 +4217,21 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) static int unicode_fill_utf8(PyObject *unicode); + +static int +unicode_ensure_utf8(PyObject *unicode) +{ + int err = 0; + if (PyUnicode_UTF8(unicode) == NULL) { + Py_BEGIN_CRITICAL_SECTION(unicode); + if (PyUnicode_UTF8(unicode) == NULL) { + err = unicode_fill_utf8(unicode); + } + Py_END_CRITICAL_SECTION(); + } + return err; +} + const char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { @@ -4227,13 +4243,11 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) return NULL; } - if (PyUnicode_UTF8(unicode) == NULL) { - if (unicode_fill_utf8(unicode) == -1) { - if (psize) { - *psize = -1; - } - return NULL; + if (unicode_ensure_utf8(unicode) == -1) { + if (psize) { + *psize = -1; } + return NULL; } if (psize) { @@ -5854,6 +5868,7 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, static int unicode_fill_utf8(PyObject *unicode) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(unicode); /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ assert(!PyUnicode_IS_ASCII(unicode)); @@ -5895,10 +5910,10 @@ unicode_fill_utf8(PyObject *unicode) PyErr_NoMemory(); return -1; } - PyUnicode_SET_UTF8(unicode, cache); - PyUnicode_SET_UTF8_LENGTH(unicode, len); memcpy(cache, start, len); cache[len] = '\0'; + PyUnicode_SET_UTF8_LENGTH(unicode, len); + PyUnicode_SET_UTF8(unicode, cache); _PyBytesWriter_Dealloc(&writer); return 0; } From 19c5134d57764d3db7b1cacec4f090c74849a5c1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 19 Dec 2024 18:15:36 +0530 Subject: [PATCH 04/33] gh-122706: fix docs for asyncio ssl sockets (#128092) --- Doc/library/ssl.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index b7fb1fc07d199f..f07d151a885692 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2508,8 +2508,8 @@ thus several things you need to be aware of: .. seealso:: The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and + ` and provides a higher level :ref:`Streams API `. + It polls for events using the :mod:`selectors` module and handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously as well. From ea578fc6d310c85538aefbb900a326c5c3424dd5 Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Thu, 19 Dec 2024 21:51:21 +0800 Subject: [PATCH 05/33] gh-127688: Add `SCHED_DEADLINE` and `SCHED_NORMAL` constants to `os` module (GH-127689) --- Doc/library/os.rst | 12 ++++++++++++ Doc/whatsnew/3.14.rst | 4 ++++ .../2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst | 2 ++ Modules/posixmodule.c | 10 ++++++++++ configure | 6 ++++++ configure.ac | 2 +- pyconfig.h.in | 3 +++ 7 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index dfe5ef0726ff7d..69e6192038ab2b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5420,10 +5420,22 @@ operating system. Scheduling policy for CPU-intensive processes that tries to preserve interactivity on the rest of the computer. +.. data:: SCHED_DEADLINE + + Scheduling policy for tasks with deadline constraints. + + .. versionadded:: next + .. data:: SCHED_IDLE Scheduling policy for extremely low priority background tasks. +.. data:: SCHED_NORMAL + + Alias for :data:`SCHED_OTHER`. + + .. versionadded:: next + .. data:: SCHED_SPORADIC Scheduling policy for sporadic server programs. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 342456cbc397f3..2e43dce5e061b4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -525,6 +525,10 @@ os same process. (Contributed by Victor Stinner in :gh:`120057`.) +* Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants + to the :mod:`os` module. + (Contributed by James Roy in :gh:`127688`.) + pathlib ------- diff --git a/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst b/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst new file mode 100644 index 00000000000000..a22b136da72faf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst @@ -0,0 +1,2 @@ +Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants +to the :mod:`os` module. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2045c6065b8e7a..151d469983fafb 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -311,6 +311,10 @@ corresponding Unix manual entries for more information on calls."); # include #endif +#ifdef HAVE_LINUX_SCHED_H +# include +#endif + #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) # undef HAVE_SCHED_SETAFFINITY #endif @@ -17523,9 +17527,15 @@ all_ins(PyObject *m) #ifdef SCHED_OTHER if (PyModule_AddIntMacro(m, SCHED_OTHER)) return -1; #endif +#ifdef SCHED_DEADLINE + if (PyModule_AddIntMacro(m, SCHED_DEADLINE)) return -1; +#endif #ifdef SCHED_FIFO if (PyModule_AddIntMacro(m, SCHED_FIFO)) return -1; #endif +#ifdef SCHED_NORMAL + if (PyModule_AddIntMacro(m, SCHED_NORMAL)) return -1; +#endif #ifdef SCHED_RR if (PyModule_AddIntMacro(m, SCHED_RR)) return -1; #endif diff --git a/configure b/configure index 6df1116fc600f2..e59c7046305d46 100755 --- a/configure +++ b/configure @@ -10984,6 +10984,12 @@ if test "x$ac_cv_header_linux_soundcard_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_SOUNDCARD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/sched.h" "ac_cv_header_linux_sched_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_sched_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_SCHED_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/tipc.h" "ac_cv_header_linux_tipc_h" "$ac_includes_default" if test "x$ac_cv_header_linux_tipc_h" = xyes diff --git a/configure.ac b/configure.ac index 8295b59b8e45fb..074e2ce3dd3024 100644 --- a/configure.ac +++ b/configure.ac @@ -2931,7 +2931,7 @@ AC_DEFINE([STDC_HEADERS], [1], AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ - linux/netfilter_ipv4.h linux/random.h linux/soundcard.h \ + linux/netfilter_ipv4.h linux/random.h linux/soundcard.h linux/sched.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 166c195a8c66fc..1ca83fd2f2ca1b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -744,6 +744,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_RANDOM_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_SCHED_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_SOUNDCARD_H From b5d1e4552f0ba40d8380368e1b099261686a89cf Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 19 Dec 2024 09:00:30 -0500 Subject: [PATCH 06/33] gh-128083: Fix macro redefinition warning in clinic. (GH-127950) --- Lib/test/clinic.test.c | 69 ++++++++++++++++++++++++++++ Modules/_io/clinic/textio.c.h | 11 ++++- Modules/clinic/_ssl.c.h | 50 +++++++++++++++++++- Tools/clinic/libclinic/parse_args.py | 3 ++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index b6ae04ecf2f8ed..e4f146c0841188 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5358,6 +5358,75 @@ static int Test_property_set_impl(TestObj *self, PyObject *value) /*[clinic end generated code: output=e4342fe9bb1d7817 input=3bc3f46a23c83a88]*/ +/*[clinic input] +@setter +Test.setter_first_with_docstr +[clinic start generated code]*/ + +#if !defined(Test_setter_first_with_docstr_DOCSTR) +# define Test_setter_first_with_docstr_DOCSTR NULL +#endif +#if defined(TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF) +# undef TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, (setter)Test_setter_first_with_docstr_set, Test_setter_first_with_docstr_DOCSTR}, +#else +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", NULL, (setter)Test_setter_first_with_docstr_set, NULL}, +#endif + +static int +Test_setter_first_with_docstr_set_impl(TestObj *self, PyObject *value); + +static int +Test_setter_first_with_docstr_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = Test_setter_first_with_docstr_set_impl(self, value); + + return return_value; +} + +static int +Test_setter_first_with_docstr_set_impl(TestObj *self, PyObject *value) +/*[clinic end generated code: output=e4d76b558a4061db input=31a045ce11bbe961]*/ + +/*[clinic input] +@getter +Test.setter_first_with_docstr + +my silly docstring +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_setter_first_with_docstr__doc__, +"my silly docstring"); +#if defined(Test_setter_first_with_docstr_DOCSTR) +# undef Test_setter_first_with_docstr_DOCSTR +#endif +#define Test_setter_first_with_docstr_DOCSTR Test_setter_first_with_docstr__doc__ + +#if !defined(Test_setter_first_with_docstr_DOCSTR) +# define Test_setter_first_with_docstr_DOCSTR NULL +#endif +#if defined(TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF) +# undef TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, (setter)Test_setter_first_with_docstr_set, Test_setter_first_with_docstr_DOCSTR}, +#else +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, NULL, Test_setter_first_with_docstr_DOCSTR}, +#endif + +static PyObject * +Test_setter_first_with_docstr_get_impl(TestObj *self); + +static PyObject * +Test_setter_first_with_docstr_get(TestObj *self, void *Py_UNUSED(context)) +{ + return Test_setter_first_with_docstr_get_impl(self); +} + +static PyObject * +Test_setter_first_with_docstr_get_impl(TestObj *self) +/*[clinic end generated code: output=749a30266f9fb443 input=10af4e43b3cb34dc]*/ + /*[clinic input] output push output preset buffer diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 160f80ada43660..0acc1f060c811b 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -208,6 +208,9 @@ PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, "Encoding of the text stream.\n" "\n" "Subclasses should override."); +#if defined(_io__TextIOBase_encoding_DOCSTR) +# undef _io__TextIOBase_encoding_DOCSTR +#endif #define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ #if !defined(_io__TextIOBase_encoding_DOCSTR) @@ -235,6 +238,9 @@ PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, "Only line endings translated during reading are considered.\n" "\n" "Subclasses should override."); +#if defined(_io__TextIOBase_newlines_DOCSTR) +# undef _io__TextIOBase_newlines_DOCSTR +#endif #define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ #if !defined(_io__TextIOBase_newlines_DOCSTR) @@ -260,6 +266,9 @@ PyDoc_STRVAR(_io__TextIOBase_errors__doc__, "The error setting of the decoder or encoder.\n" "\n" "Subclasses should override."); +#if defined(_io__TextIOBase_errors_DOCSTR) +# undef _io__TextIOBase_errors_DOCSTR +#endif #define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ #if !defined(_io__TextIOBase_errors_DOCSTR) @@ -1281,4 +1290,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=1172c500a022c65d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=423a320f087792b9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 1ff85e32ffe5a0..becdb9cc1831fa 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -264,6 +264,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, "This is typically used from within a callback function set by the sni_callback\n" "on the SSLContext to change the certificate information associated with the\n" "SSLSocket before the cryptographic exchange handshake messages."); +#if defined(_ssl__SSLSocket_context_DOCSTR) +# undef _ssl__SSLSocket_context_DOCSTR +#endif #define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ #if !defined(_ssl__SSLSocket_context_DOCSTR) @@ -318,6 +321,9 @@ _ssl__SSLSocket_context_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_server_side__doc__, "Whether this is a server-side socket."); +#if defined(_ssl__SSLSocket_server_side_DOCSTR) +# undef _ssl__SSLSocket_server_side_DOCSTR +#endif #define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ #if !defined(_ssl__SSLSocket_server_side_DOCSTR) @@ -347,6 +353,9 @@ _ssl__SSLSocket_server_side_get(PySSLSocket *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl__SSLSocket_server_hostname__doc__, "The currently set server hostname (for SNI)."); +#if defined(_ssl__SSLSocket_server_hostname_DOCSTR) +# undef _ssl__SSLSocket_server_hostname_DOCSTR +#endif #define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ #if !defined(_ssl__SSLSocket_server_hostname_DOCSTR) @@ -378,6 +387,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_owner__doc__, "The Python-level owner of this object.\n" "\n" "Passed as \"self\" in servername callback."); +#if defined(_ssl__SSLSocket_owner_DOCSTR) +# undef _ssl__SSLSocket_owner_DOCSTR +#endif #define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ #if !defined(_ssl__SSLSocket_owner_DOCSTR) @@ -668,6 +680,9 @@ _ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNU PyDoc_STRVAR(_ssl__SSLSocket_session__doc__, "The underlying SSLSession object."); +#if defined(_ssl__SSLSocket_session_DOCSTR) +# undef _ssl__SSLSocket_session_DOCSTR +#endif #define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ #if !defined(_ssl__SSLSocket_session_DOCSTR) @@ -722,6 +737,9 @@ _ssl__SSLSocket_session_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_session_reused__doc__, "Was the client session reused during handshake?"); +#if defined(_ssl__SSLSocket_session_reused_DOCSTR) +# undef _ssl__SSLSocket_session_reused_DOCSTR +#endif #define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ #if !defined(_ssl__SSLSocket_session_reused_DOCSTR) @@ -1077,6 +1095,9 @@ _ssl__SSLContext_maximum_version_set(PySSLContext *self, PyObject *value, void * PyDoc_STRVAR(_ssl__SSLContext_num_tickets__doc__, "Control the number of TLSv1.3 session tickets."); +#if defined(_ssl__SSLContext_num_tickets_DOCSTR) +# undef _ssl__SSLContext_num_tickets_DOCSTR +#endif #define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ #if !defined(_ssl__SSLContext_num_tickets_DOCSTR) @@ -1131,6 +1152,9 @@ _ssl__SSLContext_num_tickets_set(PySSLContext *self, PyObject *value, void *Py_U PyDoc_STRVAR(_ssl__SSLContext_security_level__doc__, "The current security level."); +#if defined(_ssl__SSLContext_security_level_DOCSTR) +# undef _ssl__SSLContext_security_level_DOCSTR +#endif #define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ #if !defined(_ssl__SSLContext_security_level_DOCSTR) @@ -1778,6 +1802,9 @@ PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, "with the SSLSocket, the server name as a string, and the SSLContext object.\n" "\n" "See RFC 6066 for details of the SNI extension."); +#if defined(_ssl__SSLContext_sni_callback_DOCSTR) +# undef _ssl__SSLContext_sni_callback_DOCSTR +#endif #define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ #if !defined(_ssl__SSLContext_sni_callback_DOCSTR) @@ -2100,6 +2127,9 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyDoc_STRVAR(_ssl_MemoryBIO_pending__doc__, "The number of bytes pending in the memory BIO."); +#if defined(_ssl_MemoryBIO_pending_DOCSTR) +# undef _ssl_MemoryBIO_pending_DOCSTR +#endif #define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ #if !defined(_ssl_MemoryBIO_pending_DOCSTR) @@ -2129,6 +2159,9 @@ _ssl_MemoryBIO_pending_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_MemoryBIO_eof__doc__, "Whether the memory BIO is at EOF."); +#if defined(_ssl_MemoryBIO_eof_DOCSTR) +# undef _ssl_MemoryBIO_eof_DOCSTR +#endif #define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ #if !defined(_ssl_MemoryBIO_eof_DOCSTR) @@ -2262,6 +2295,9 @@ _ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(_ssl_SSLSession_time__doc__, "Session creation time (seconds since epoch)."); +#if defined(_ssl_SSLSession_time_DOCSTR) +# undef _ssl_SSLSession_time_DOCSTR +#endif #define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ #if !defined(_ssl_SSLSession_time_DOCSTR) @@ -2291,6 +2327,9 @@ _ssl_SSLSession_time_get(PySSLSession *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_SSLSession_timeout__doc__, "Session timeout (delta in seconds)."); +#if defined(_ssl_SSLSession_timeout_DOCSTR) +# undef _ssl_SSLSession_timeout_DOCSTR +#endif #define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ #if !defined(_ssl_SSLSession_timeout_DOCSTR) @@ -2320,6 +2359,9 @@ _ssl_SSLSession_timeout_get(PySSLSession *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_SSLSession_ticket_lifetime_hint__doc__, "Ticket life time hint."); +#if defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) +# undef _ssl_SSLSession_ticket_lifetime_hint_DOCSTR +#endif #define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ #if !defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) @@ -2349,6 +2391,9 @@ _ssl_SSLSession_ticket_lifetime_hint_get(PySSLSession *self, void *Py_UNUSED(con PyDoc_STRVAR(_ssl_SSLSession_id__doc__, "Session ID."); +#if defined(_ssl_SSLSession_id_DOCSTR) +# undef _ssl_SSLSession_id_DOCSTR +#endif #define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ #if !defined(_ssl_SSLSession_id_DOCSTR) @@ -2378,6 +2423,9 @@ _ssl_SSLSession_id_get(PySSLSession *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_SSLSession_has_ticket__doc__, "Does the session contain a ticket?"); +#if defined(_ssl_SSLSession_has_ticket_DOCSTR) +# undef _ssl_SSLSession_has_ticket_DOCSTR +#endif #define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ #if !defined(_ssl_SSLSession_has_ticket_DOCSTR) @@ -2830,4 +2878,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=654d6d7af659f6cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e71f1ef621aead08 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index a57d729bec5733..ff4731e99b98d4 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -146,6 +146,9 @@ def declare_parser( GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({getset_basename}__doc__, {docstring}); + #if defined({getset_basename}_DOCSTR) + # undef {getset_basename}_DOCSTR + #endif #define {getset_basename}_DOCSTR {getset_basename}__doc__ """) IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" From b9b3e4a076caddf7876d1d4d762a117a26faffcf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 19 Dec 2024 15:45:34 +0100 Subject: [PATCH 07/33] gh-127951: Add build option to enable pystats on Windows (GH-127952) --- .../next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst | 1 + PCbuild/build.bat | 3 +++ PCbuild/pyproject.props | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst b/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst new file mode 100644 index 00000000000000..0c1df0e6bd7baa --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst @@ -0,0 +1 @@ +Add option ``--pystats`` to the Windows build to enable performance statistics collection. diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 6d3ce81651ade5..d3e3894c203a65 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -39,6 +39,7 @@ echo. --regen Regenerate all opcodes, grammar and tokens. echo. --experimental-jit Enable the experimental just-in-time compiler. echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. +echo. --pystats Enable PyStats collection. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -93,6 +94,7 @@ if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & go if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts +if "%~1"=="--pystats" (set PyStats=1) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -186,6 +188,7 @@ echo on /p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseJIT=%UseJIT%^ /p:UseTIER2=%UseTIER2%^ + /p:PyStats=%PyStats%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index c65341179376ea..17abfa85201a90 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -42,6 +42,8 @@ <_DebugPreprocessorDefinition>NDEBUG; <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG; + <_PyStatsPreprocessorDefinition>PyStats; + <_PyStatsPreprocessorDefinition Condition="$(PySTATS) != ''">Py_STATS; <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) @@ -50,7 +52,7 @@ $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) - WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed From 7b811d0562a0bf7433165785f1549ac199610f8b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 19 Dec 2024 10:17:15 -0500 Subject: [PATCH 08/33] gh-128008: Add `PyWeakref_IsDead()` (GH-128009) The `PyWeakref_IsDead()` function tests if a weak reference is dead without any side effects. Although you can also detect if a weak reference is dead using `PyWeakref_GetRef()`, that function returns a strong reference that must be `Py_DECREF()`'d, which can introduce side effects if the last reference is concurrently dropped (at least in the free threading build). --- Doc/c-api/weakref.rst | 9 +++++++++ Include/cpython/weakrefobject.h | 3 +++ .../2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst | 2 ++ Modules/_testcapimodule.c | 14 ++++++++++++++ Objects/weakrefobject.c | 13 +++++++++++++ 5 files changed, 41 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 8f233e16fb17cf..c3c6cf413dcef5 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -88,6 +88,15 @@ as much as it can. Use :c:func:`PyWeakref_GetRef` instead. +.. c:function:: int PyWeakref_IsDead(PyObject *ref) + + Test if the weak reference *ref* is dead. Returns 1 if the reference is + dead, 0 if it is alive, and -1 with an error set if *ref* is not a weak + reference object. + + .. versionadded:: 3.14 + + .. c:function:: void PyObject_ClearWeakRefs(PyObject *object) This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 9aa1a92c413fe9..da8e77cddaca63 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -45,6 +45,9 @@ PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); #define _PyWeakref_CAST(op) \ (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) +// Test if a weak reference is dead. +PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { PyWeakReference *ref = _PyWeakref_CAST(ref_obj); diff --git a/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst b/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst new file mode 100644 index 00000000000000..2349eccac8fedc --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyWeakref_IsDead` function, which tests if a weak reference is +dead. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 8d86b535effb9a..f737250ac29d57 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3144,6 +3144,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyObject *ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(weakref, &ref) == 1); assert(ref == obj); + assert(!PyWeakref_IsDead(weakref)); assert(Py_REFCNT(obj) == (refcnt + 1)); Py_DECREF(ref); @@ -3159,6 +3160,8 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(Py_REFCNT(obj) == 1); Py_DECREF(obj); + assert(PyWeakref_IsDead(weakref)); + // test PyWeakref_GET_OBJECT(), reference is dead assert(PyWeakref_GET_OBJECT(weakref) == Py_None); @@ -3181,6 +3184,12 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyErr_Clear(); assert(ref == NULL); + // test PyWeakRef_IsDead(), invalid type + assert(!PyErr_Occurred()); + assert(PyWeakref_IsDead(invalid_weakref) == -1); + assert(PyErr_ExceptionMatches(PyExc_TypeError)); + PyErr_Clear(); + // test PyWeakref_GetObject(), invalid type assert(PyWeakref_GetObject(invalid_weakref) == NULL); assert(PyErr_ExceptionMatches(PyExc_SystemError)); @@ -3193,6 +3202,11 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(ref == NULL); PyErr_Clear(); + // test PyWeakref_IsDead(NULL) + assert(PyWeakref_IsDead(NULL) == -1); + assert(PyErr_ExceptionMatches(PyExc_SystemError)); + PyErr_Clear(); + // test PyWeakref_GetObject(NULL) assert(PyWeakref_GetObject(NULL) == NULL); assert(PyErr_ExceptionMatches(PyExc_SystemError)); diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 9e3da1c3394d5b..0ee64ed70a63cd 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -932,6 +932,19 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) return (PyObject *)get_or_create_weakref(type, ob, callback); } +int +PyWeakref_IsDead(PyObject *ref) +{ + if (ref == NULL) { + PyErr_BadInternalCall(); + return -1; + } + if (!PyWeakref_Check(ref)) { + PyErr_Format(PyExc_TypeError, "expected a weakref, got %T", ref); + return -1; + } + return _PyWeakref_IS_DEAD(ref); +} int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) From d2f1d917e8b3d2dd8f35495c7632a32688883332 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 19 Dec 2024 16:59:51 +0000 Subject: [PATCH 09/33] GH-122548: Implement branch taken and not taken events for sys.monitoring (GH-122564) --- Doc/c-api/monitoring.rst | 12 +- Doc/library/sys.monitoring.rst | 29 +- Doc/whatsnew/3.13.rst | 2 +- Doc/whatsnew/3.14.rst | 14 + Include/cpython/code.h | 6 +- Include/cpython/monitoring.h | 43 +- Include/internal/pycore_code.h | 2 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 25 +- Include/internal/pycore_opcode_utils.h | 6 + Include/internal/pycore_uop_ids.h | 1 + Include/opcode_ids.h | 198 ++++----- Lib/_opcode_metadata.py | 198 ++++----- Lib/test/test_compiler_codegen.py | 2 + Lib/test/test_dis.py | 391 +++++++++--------- Lib/test/test_monitoring.py | 67 ++- ...-12-13-14-21-04.gh-issue-122548.hq3Vud.rst | 4 + Modules/_testcapi/monitoring.c | 25 +- Objects/codeobject.c | 7 + Programs/test_frozenmain.h | 66 +-- Python/bytecodes.c | 60 ++- Python/codegen.c | 9 +- Python/executor_cases.c.h | 2 + Python/flowgraph.c | 6 + Python/generated_cases.c.h | 74 ++-- Python/instrumentation.c | 321 +++++++++++--- Python/opcode_targets.h | 4 +- Python/optimizer_cases.c.h | 2 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 + 29 files changed, 998 insertions(+), 583 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 51d866cfd47469..bda6cd271197d0 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -75,9 +75,14 @@ See :mod:`sys.monitoring` for descriptions of the events. Fire a ``JUMP`` event. -.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) +.. c:function:: int PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) - Fire a ``BRANCH`` event. + Fire a ``BRANCH_LEFT`` event. + + +.. c:function:: int PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_RIGHT`` event. .. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) @@ -168,7 +173,8 @@ would typically correspond to a python function. ================================================== ===================================== Macro Event ================================================== ===================================== - .. c:macro:: PY_MONITORING_EVENT_BRANCH :monitoring-event:`BRANCH` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_LEFT :monitoring-event:`BRANCH_LEFT` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_RIGHT :monitoring-event:`BRANCH_RIGHT` .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index f7140af2494898..cfdcdf2e2df476 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -79,9 +79,17 @@ Events The following events are supported: -.. monitoring-event:: BRANCH +.. monitoring-event:: BRANCH_LEFT - A conditional branch is taken (or not). + A conditional branch goes left. + + It is up to the tool to determine how to present "left" and "right" branches. + There is no guarantee which branch is "left" and which is "right", except + that it will be consistent for the duration of the program. + +.. monitoring-event:: BRANCH_RIGHT + + A conditional branch goes right. .. monitoring-event:: CALL @@ -180,9 +188,20 @@ The local events are: * :monitoring-event:`LINE` * :monitoring-event:`INSTRUCTION` * :monitoring-event:`JUMP` -* :monitoring-event:`BRANCH` +* :monitoring-event:`BRANCH_LEFT` +* :monitoring-event:`BRANCH_RIGHT` * :monitoring-event:`STOP_ITERATION` +Deprecated event +'''''''''''''''' + +* ``BRANCH`` + +The ``BRANCH`` event is deprecated in 3.14. +Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` +events will give much better performance as they can be disabled +independently. + Ancillary events '''''''''''''''' @@ -357,13 +376,11 @@ Different events will provide the callback function with different arguments, as func(code: CodeType, line_number: int) -> DISABLE | Any -* :monitoring-event:`BRANCH` and :monitoring-event:`JUMP`:: +* :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any Note that the *destination_offset* is where the code will next execute. - For an untaken branch this will be the offset of the instruction following - the branch. * :monitoring-event:`INSTRUCTION`:: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9f6d98b9950d19..45cc1b5bad9b18 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1971,7 +1971,7 @@ New Features * :c:func:`PyMonitoring_FireCallEvent` * :c:func:`PyMonitoring_FireLineEvent` * :c:func:`PyMonitoring_FireJumpEvent` - * :c:func:`PyMonitoring_FireBranchEvent` + * ``PyMonitoring_FireBranchEvent`` * :c:func:`PyMonitoring_FireCReturnEvent` * :c:func:`PyMonitoring_FirePyThrowEvent` * :c:func:`PyMonitoring_FireRaiseEvent` diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2e43dce5e061b4..97a37a82f76b9b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -603,6 +603,11 @@ sys which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. +sys.monitoring +-------------- + +Two new events are added: :monitoring-event:`BRANCH_LEFT` and +:monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. tkinter ------- @@ -1144,6 +1149,11 @@ New features a :exc:`UnicodeError` object. (Contributed by Bénédikt Tran in :gh:`127691`.) +* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent` for generating + :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` + events, respectively. + Porting to Python 3.14 ---------------------- @@ -1177,6 +1187,10 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-future.rst +* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should + be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` + and :c:func:`PyMonitoring_FireBranchRightEvent`. + Removed ------- diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 3899d4269233a1..c3c0165d556ead 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -11,11 +11,11 @@ extern "C" { /* Total tool ids available */ #define _PY_MONITORING_TOOL_IDS 8 /* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 10 +#define _PY_MONITORING_LOCAL_EVENTS 11 /* Count of all "real" monitoring events (not derived from other events) */ -#define _PY_MONITORING_UNGROUPED_EVENTS 15 +#define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ -#define _PY_MONITORING_EVENTS 17 +#define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 797ba51246b1c6..ce92942404c9f7 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -13,25 +13,27 @@ #define PY_MONITORING_EVENT_LINE 5 #define PY_MONITORING_EVENT_INSTRUCTION 6 #define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 +#define PY_MONITORING_EVENT_BRANCH_LEFT 8 +#define PY_MONITORING_EVENT_BRANCH_RIGHT 9 +#define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ ((ev) < _PY_MONITORING_LOCAL_EVENTS) /* Other events, mainly exceptions */ -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 +#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 +#define PY_MONITORING_EVENT_PY_UNWIND 13 +#define PY_MONITORING_EVENT_PY_THROW 14 +#define PY_MONITORING_EVENT_RERAISE 15 /* Ancillary events */ -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 +#define PY_MONITORING_EVENT_C_RETURN 16 +#define PY_MONITORING_EVENT_C_RAISE 17 +#define PY_MONITORING_EVENT_BRANCH 18 typedef struct _PyMonitoringState { @@ -74,10 +76,18 @@ PyAPI_FUNC(int) _PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset); -PyAPI_FUNC(int) +Py_DEPRECATED(3.14) PyAPI_FUNC(int) _PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset); +PyAPI_FUNC(int) +_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + PyAPI_FUNC(int) _PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval); @@ -174,12 +184,21 @@ PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t } static inline int -PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, +PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchRightEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) { _PYMONITORING_IF_ACTIVE( state, - _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); + _PyMonitoring_FireBranchLeftEvent(state, codelike, offset, target_offset)); } static inline int diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index d607a54aa4a2f5..d97fe81a2fc54a 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -603,6 +603,8 @@ extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); + struct _PyCode8 _PyCode_DEF(8); PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 14e29576875c6e..079a9befcd4c5e 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -262,6 +262,7 @@ Known values: Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) Python 3.14a1 3608 (Add support for slices) Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) + Python 3.14a3 3610 (Add NOT_TAKEN instruction) Python 3.15 will start with 3650 @@ -274,7 +275,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3609 +#define PYC_MAGIC_NUMBER 3611 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d2ae8928a8fe8f..e09fff062b5202 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -243,6 +243,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: return 0; + case INSTRUMENTED_NOT_TAKEN: + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -367,6 +369,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case NOP: return 0; + case NOT_TAKEN: + return 0; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -702,6 +706,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: return 0; + case INSTRUMENTED_NOT_TAKEN: + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -826,6 +832,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case NOP: return 0; + case NOT_TAKEN: + return 0; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -1387,6 +1395,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 0; return 0; } + case INSTRUMENTED_NOT_TAKEN: { + *effect = 0; + return 0; + } case INSTRUMENTED_POP_JUMP_IF_FALSE: { *effect = 0; return 0; @@ -1635,6 +1647,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 0; return 0; } + case NOT_TAKEN: { + *effect = 0; + return 0; + } case POP_BLOCK: { *effect = 0; return 0; @@ -2043,6 +2059,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IXC, 0 }, + [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, @@ -2100,6 +2117,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -2304,6 +2322,7 @@ _PyOpcode_macro_expansion[256] = { [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, 0, 0 } } }, [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, 0, 0 } } }, [NOP] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, + [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, 0, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 9, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 9, 1 } } }, @@ -2462,6 +2481,7 @@ const char *_PyOpcode_OpName[266] = { [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_NOT_TAKEN] = "INSTRUMENTED_NOT_TAKEN", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", @@ -2524,6 +2544,7 @@ const char *_PyOpcode_OpName[266] = { [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [NOP] = "NOP", + [NOT_TAKEN] = "NOT_TAKEN", [POP_BLOCK] = "POP_BLOCK", [POP_EXCEPT] = "POP_EXCEPT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", @@ -2718,6 +2739,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, @@ -2775,6 +2797,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [MATCH_MAPPING] = MATCH_MAPPING, [MATCH_SEQUENCE] = MATCH_SEQUENCE, [NOP] = NOP, + [NOT_TAKEN] = NOT_TAKEN, [POP_EXCEPT] = POP_EXCEPT, [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, @@ -2833,7 +2856,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 116: \ case 117: \ case 118: \ case 119: \ @@ -2874,7 +2896,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 233: \ case 234: \ case 235: \ - case 236: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index c6ce7e65a65d4b..0872231d1f2d11 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -45,6 +45,12 @@ extern "C" { (opcode) == JUMP_BACKWARD || \ (opcode) == JUMP_BACKWARD_NO_INTERRUPT) +#define IS_CONDITIONAL_JUMP_OPCODE(opcode) \ + ((opcode) == POP_JUMP_IF_FALSE || \ + (opcode) == POP_JUMP_IF_TRUE || \ + (opcode) == POP_JUMP_IF_NONE || \ + (opcode) == POP_JUMP_IF_NOT_NONE) + #define IS_SCOPE_EXIT_OPCODE(opcode) \ ((opcode) == RETURN_VALUE || \ (opcode) == RAISE_VARARGS || \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 45563585dd5681..5fc57e48f500d0 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -155,6 +155,7 @@ extern "C" { #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD #define _INSTRUMENTED_LINE INSTRUMENTED_LINE #define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_NOT_TAKEN INSTRUMENTED_NOT_TAKEN #define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index ce3d23eaa6d56d..3cd189b93dd9d6 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -38,94 +38,95 @@ extern "C" { #define MATCH_MAPPING 25 #define MATCH_SEQUENCE 26 #define NOP 27 -#define POP_EXCEPT 28 -#define POP_TOP 29 -#define PUSH_EXC_INFO 30 -#define PUSH_NULL 31 -#define RETURN_GENERATOR 32 -#define RETURN_VALUE 33 -#define SETUP_ANNOTATIONS 34 -#define STORE_SLICE 35 -#define STORE_SUBSCR 36 -#define TO_BOOL 37 -#define UNARY_INVERT 38 -#define UNARY_NEGATIVE 39 -#define UNARY_NOT 40 -#define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_LIST 43 -#define BUILD_MAP 44 -#define BUILD_SET 45 -#define BUILD_SLICE 46 -#define BUILD_STRING 47 -#define BUILD_TUPLE 48 -#define CALL 49 -#define CALL_FUNCTION_EX 50 -#define CALL_INTRINSIC_1 51 -#define CALL_INTRINSIC_2 52 -#define CALL_KW 53 -#define COMPARE_OP 54 -#define CONTAINS_OP 55 -#define CONVERT_VALUE 56 -#define COPY 57 -#define COPY_FREE_VARS 58 -#define DELETE_ATTR 59 -#define DELETE_DEREF 60 -#define DELETE_FAST 61 -#define DELETE_GLOBAL 62 -#define DELETE_NAME 63 -#define DICT_MERGE 64 -#define DICT_UPDATE 65 -#define EXTENDED_ARG 66 -#define FOR_ITER 67 -#define GET_AWAITABLE 68 -#define IMPORT_FROM 69 -#define IMPORT_NAME 70 -#define IS_OP 71 -#define JUMP_BACKWARD 72 -#define JUMP_BACKWARD_NO_INTERRUPT 73 -#define JUMP_FORWARD 74 -#define LIST_APPEND 75 -#define LIST_EXTEND 76 -#define LOAD_ATTR 77 -#define LOAD_COMMON_CONSTANT 78 -#define LOAD_CONST 79 -#define LOAD_DEREF 80 -#define LOAD_FAST 81 -#define LOAD_FAST_AND_CLEAR 82 -#define LOAD_FAST_CHECK 83 -#define LOAD_FAST_LOAD_FAST 84 -#define LOAD_FROM_DICT_OR_DEREF 85 -#define LOAD_FROM_DICT_OR_GLOBALS 86 -#define LOAD_GLOBAL 87 -#define LOAD_NAME 88 -#define LOAD_SMALL_INT 89 -#define LOAD_SPECIAL 90 -#define LOAD_SUPER_ATTR 91 -#define MAKE_CELL 92 -#define MAP_ADD 93 -#define MATCH_CLASS 94 -#define POP_JUMP_IF_FALSE 95 -#define POP_JUMP_IF_NONE 96 -#define POP_JUMP_IF_NOT_NONE 97 -#define POP_JUMP_IF_TRUE 98 -#define RAISE_VARARGS 99 -#define RERAISE 100 -#define SEND 101 -#define SET_ADD 102 -#define SET_FUNCTION_ATTRIBUTE 103 -#define SET_UPDATE 104 -#define STORE_ATTR 105 -#define STORE_DEREF 106 -#define STORE_FAST 107 -#define STORE_FAST_LOAD_FAST 108 -#define STORE_FAST_STORE_FAST 109 -#define STORE_GLOBAL 110 -#define STORE_NAME 111 -#define SWAP 112 -#define UNPACK_EX 113 -#define UNPACK_SEQUENCE 114 -#define YIELD_VALUE 115 +#define NOT_TAKEN 28 +#define POP_EXCEPT 29 +#define POP_TOP 30 +#define PUSH_EXC_INFO 31 +#define PUSH_NULL 32 +#define RETURN_GENERATOR 33 +#define RETURN_VALUE 34 +#define SETUP_ANNOTATIONS 35 +#define STORE_SLICE 36 +#define STORE_SUBSCR 37 +#define TO_BOOL 38 +#define UNARY_INVERT 39 +#define UNARY_NEGATIVE 40 +#define UNARY_NOT 41 +#define WITH_EXCEPT_START 42 +#define BINARY_OP 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_FUNCTION_EX 51 +#define CALL_INTRINSIC_1 52 +#define CALL_INTRINSIC_2 53 +#define CALL_KW 54 +#define COMPARE_OP 55 +#define CONTAINS_OP 56 +#define CONVERT_VALUE 57 +#define COPY 58 +#define COPY_FREE_VARS 59 +#define DELETE_ATTR 60 +#define DELETE_DEREF 61 +#define DELETE_FAST 62 +#define DELETE_GLOBAL 63 +#define DELETE_NAME 64 +#define DICT_MERGE 65 +#define DICT_UPDATE 66 +#define EXTENDED_ARG 67 +#define FOR_ITER 68 +#define GET_AWAITABLE 69 +#define IMPORT_FROM 70 +#define IMPORT_NAME 71 +#define IS_OP 72 +#define JUMP_BACKWARD 73 +#define JUMP_BACKWARD_NO_INTERRUPT 74 +#define JUMP_FORWARD 75 +#define LIST_APPEND 76 +#define LIST_EXTEND 77 +#define LOAD_ATTR 78 +#define LOAD_COMMON_CONSTANT 79 +#define LOAD_CONST 80 +#define LOAD_DEREF 81 +#define LOAD_FAST 82 +#define LOAD_FAST_AND_CLEAR 83 +#define LOAD_FAST_CHECK 84 +#define LOAD_FAST_LOAD_FAST 85 +#define LOAD_FROM_DICT_OR_DEREF 86 +#define LOAD_FROM_DICT_OR_GLOBALS 87 +#define LOAD_GLOBAL 88 +#define LOAD_NAME 89 +#define LOAD_SMALL_INT 90 +#define LOAD_SPECIAL 91 +#define LOAD_SUPER_ATTR 92 +#define MAKE_CELL 93 +#define MAP_ADD 94 +#define MATCH_CLASS 95 +#define POP_JUMP_IF_FALSE 96 +#define POP_JUMP_IF_NONE 97 +#define POP_JUMP_IF_NOT_NONE 98 +#define POP_JUMP_IF_TRUE 99 +#define RAISE_VARARGS 100 +#define RERAISE 101 +#define SEND 102 +#define SET_ADD 103 +#define SET_FUNCTION_ATTRIBUTE 104 +#define SET_UPDATE 105 +#define STORE_ATTR 106 +#define STORE_DEREF 107 +#define STORE_FAST 108 +#define STORE_FAST_LOAD_FAST 109 +#define STORE_FAST_STORE_FAST 110 +#define STORE_GLOBAL 111 +#define STORE_NAME 112 +#define SWAP 113 +#define UNPACK_EX 114 +#define UNPACK_SEQUENCE 115 +#define YIELD_VALUE 116 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -205,14 +206,15 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 225 #define UNPACK_SEQUENCE_TUPLE 226 #define UNPACK_SEQUENCE_TWO_TUPLE 227 -#define INSTRUMENTED_END_FOR 237 -#define INSTRUMENTED_END_SEND 238 -#define INSTRUMENTED_LOAD_SUPER_ATTR 239 -#define INSTRUMENTED_FOR_ITER 240 -#define INSTRUMENTED_CALL_KW 241 -#define INSTRUMENTED_CALL_FUNCTION_EX 242 -#define INSTRUMENTED_INSTRUCTION 243 -#define INSTRUMENTED_JUMP_FORWARD 244 +#define INSTRUMENTED_END_FOR 236 +#define INSTRUMENTED_END_SEND 237 +#define INSTRUMENTED_LOAD_SUPER_ATTR 238 +#define INSTRUMENTED_FOR_ITER 239 +#define INSTRUMENTED_CALL_KW 240 +#define INSTRUMENTED_CALL_FUNCTION_EX 241 +#define INSTRUMENTED_INSTRUCTION 242 +#define INSTRUMENTED_JUMP_FORWARD 243 +#define INSTRUMENTED_NOT_TAKEN 244 #define INSTRUMENTED_POP_JUMP_IF_TRUE 245 #define INSTRUMENTED_POP_JUMP_IF_FALSE 246 #define INSTRUMENTED_POP_JUMP_IF_NONE 247 @@ -235,9 +237,9 @@ extern "C" { #define SETUP_WITH 264 #define STORE_FAST_MAYBE_NULL 265 -#define HAVE_ARGUMENT 41 +#define HAVE_ARGUMENT 42 #define MIN_SPECIALIZED_OPCODE 150 -#define MIN_INSTRUMENTED_OPCODE 237 +#define MIN_INSTRUMENTED_OPCODE 236 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index cda3c340c322f3..dada2cb5fa033f 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -231,102 +231,104 @@ 'MATCH_MAPPING': 25, 'MATCH_SEQUENCE': 26, 'NOP': 27, - 'POP_EXCEPT': 28, - 'POP_TOP': 29, - 'PUSH_EXC_INFO': 30, - 'PUSH_NULL': 31, - 'RETURN_GENERATOR': 32, - 'RETURN_VALUE': 33, - 'SETUP_ANNOTATIONS': 34, - 'STORE_SLICE': 35, - 'STORE_SUBSCR': 36, - 'TO_BOOL': 37, - 'UNARY_INVERT': 38, - 'UNARY_NEGATIVE': 39, - 'UNARY_NOT': 40, - 'WITH_EXCEPT_START': 41, - 'BINARY_OP': 42, - 'BUILD_LIST': 43, - 'BUILD_MAP': 44, - 'BUILD_SET': 45, - 'BUILD_SLICE': 46, - 'BUILD_STRING': 47, - 'BUILD_TUPLE': 48, - 'CALL': 49, - 'CALL_FUNCTION_EX': 50, - 'CALL_INTRINSIC_1': 51, - 'CALL_INTRINSIC_2': 52, - 'CALL_KW': 53, - 'COMPARE_OP': 54, - 'CONTAINS_OP': 55, - 'CONVERT_VALUE': 56, - 'COPY': 57, - 'COPY_FREE_VARS': 58, - 'DELETE_ATTR': 59, - 'DELETE_DEREF': 60, - 'DELETE_FAST': 61, - 'DELETE_GLOBAL': 62, - 'DELETE_NAME': 63, - 'DICT_MERGE': 64, - 'DICT_UPDATE': 65, - 'EXTENDED_ARG': 66, - 'FOR_ITER': 67, - 'GET_AWAITABLE': 68, - 'IMPORT_FROM': 69, - 'IMPORT_NAME': 70, - 'IS_OP': 71, - 'JUMP_BACKWARD': 72, - 'JUMP_BACKWARD_NO_INTERRUPT': 73, - 'JUMP_FORWARD': 74, - 'LIST_APPEND': 75, - 'LIST_EXTEND': 76, - 'LOAD_ATTR': 77, - 'LOAD_COMMON_CONSTANT': 78, - 'LOAD_CONST': 79, - 'LOAD_DEREF': 80, - 'LOAD_FAST': 81, - 'LOAD_FAST_AND_CLEAR': 82, - 'LOAD_FAST_CHECK': 83, - 'LOAD_FAST_LOAD_FAST': 84, - 'LOAD_FROM_DICT_OR_DEREF': 85, - 'LOAD_FROM_DICT_OR_GLOBALS': 86, - 'LOAD_GLOBAL': 87, - 'LOAD_NAME': 88, - 'LOAD_SMALL_INT': 89, - 'LOAD_SPECIAL': 90, - 'LOAD_SUPER_ATTR': 91, - 'MAKE_CELL': 92, - 'MAP_ADD': 93, - 'MATCH_CLASS': 94, - 'POP_JUMP_IF_FALSE': 95, - 'POP_JUMP_IF_NONE': 96, - 'POP_JUMP_IF_NOT_NONE': 97, - 'POP_JUMP_IF_TRUE': 98, - 'RAISE_VARARGS': 99, - 'RERAISE': 100, - 'SEND': 101, - 'SET_ADD': 102, - 'SET_FUNCTION_ATTRIBUTE': 103, - 'SET_UPDATE': 104, - 'STORE_ATTR': 105, - 'STORE_DEREF': 106, - 'STORE_FAST': 107, - 'STORE_FAST_LOAD_FAST': 108, - 'STORE_FAST_STORE_FAST': 109, - 'STORE_GLOBAL': 110, - 'STORE_NAME': 111, - 'SWAP': 112, - 'UNPACK_EX': 113, - 'UNPACK_SEQUENCE': 114, - 'YIELD_VALUE': 115, - 'INSTRUMENTED_END_FOR': 237, - 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 239, - 'INSTRUMENTED_FOR_ITER': 240, - 'INSTRUMENTED_CALL_KW': 241, - 'INSTRUMENTED_CALL_FUNCTION_EX': 242, - 'INSTRUMENTED_INSTRUCTION': 243, - 'INSTRUMENTED_JUMP_FORWARD': 244, + 'NOT_TAKEN': 28, + 'POP_EXCEPT': 29, + 'POP_TOP': 30, + 'PUSH_EXC_INFO': 31, + 'PUSH_NULL': 32, + 'RETURN_GENERATOR': 33, + 'RETURN_VALUE': 34, + 'SETUP_ANNOTATIONS': 35, + 'STORE_SLICE': 36, + 'STORE_SUBSCR': 37, + 'TO_BOOL': 38, + 'UNARY_INVERT': 39, + 'UNARY_NEGATIVE': 40, + 'UNARY_NOT': 41, + 'WITH_EXCEPT_START': 42, + 'BINARY_OP': 43, + 'BUILD_LIST': 44, + 'BUILD_MAP': 45, + 'BUILD_SET': 46, + 'BUILD_SLICE': 47, + 'BUILD_STRING': 48, + 'BUILD_TUPLE': 49, + 'CALL': 50, + 'CALL_FUNCTION_EX': 51, + 'CALL_INTRINSIC_1': 52, + 'CALL_INTRINSIC_2': 53, + 'CALL_KW': 54, + 'COMPARE_OP': 55, + 'CONTAINS_OP': 56, + 'CONVERT_VALUE': 57, + 'COPY': 58, + 'COPY_FREE_VARS': 59, + 'DELETE_ATTR': 60, + 'DELETE_DEREF': 61, + 'DELETE_FAST': 62, + 'DELETE_GLOBAL': 63, + 'DELETE_NAME': 64, + 'DICT_MERGE': 65, + 'DICT_UPDATE': 66, + 'EXTENDED_ARG': 67, + 'FOR_ITER': 68, + 'GET_AWAITABLE': 69, + 'IMPORT_FROM': 70, + 'IMPORT_NAME': 71, + 'IS_OP': 72, + 'JUMP_BACKWARD': 73, + 'JUMP_BACKWARD_NO_INTERRUPT': 74, + 'JUMP_FORWARD': 75, + 'LIST_APPEND': 76, + 'LIST_EXTEND': 77, + 'LOAD_ATTR': 78, + 'LOAD_COMMON_CONSTANT': 79, + 'LOAD_CONST': 80, + 'LOAD_DEREF': 81, + 'LOAD_FAST': 82, + 'LOAD_FAST_AND_CLEAR': 83, + 'LOAD_FAST_CHECK': 84, + 'LOAD_FAST_LOAD_FAST': 85, + 'LOAD_FROM_DICT_OR_DEREF': 86, + 'LOAD_FROM_DICT_OR_GLOBALS': 87, + 'LOAD_GLOBAL': 88, + 'LOAD_NAME': 89, + 'LOAD_SMALL_INT': 90, + 'LOAD_SPECIAL': 91, + 'LOAD_SUPER_ATTR': 92, + 'MAKE_CELL': 93, + 'MAP_ADD': 94, + 'MATCH_CLASS': 95, + 'POP_JUMP_IF_FALSE': 96, + 'POP_JUMP_IF_NONE': 97, + 'POP_JUMP_IF_NOT_NONE': 98, + 'POP_JUMP_IF_TRUE': 99, + 'RAISE_VARARGS': 100, + 'RERAISE': 101, + 'SEND': 102, + 'SET_ADD': 103, + 'SET_FUNCTION_ATTRIBUTE': 104, + 'SET_UPDATE': 105, + 'STORE_ATTR': 106, + 'STORE_DEREF': 107, + 'STORE_FAST': 108, + 'STORE_FAST_LOAD_FAST': 109, + 'STORE_FAST_STORE_FAST': 110, + 'STORE_GLOBAL': 111, + 'STORE_NAME': 112, + 'SWAP': 113, + 'UNPACK_EX': 114, + 'UNPACK_SEQUENCE': 115, + 'YIELD_VALUE': 116, + 'INSTRUMENTED_END_FOR': 236, + 'INSTRUMENTED_END_SEND': 237, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 238, + 'INSTRUMENTED_FOR_ITER': 239, + 'INSTRUMENTED_CALL_KW': 240, + 'INSTRUMENTED_CALL_FUNCTION_EX': 241, + 'INSTRUMENTED_INSTRUCTION': 242, + 'INSTRUMENTED_JUMP_FORWARD': 243, + 'INSTRUMENTED_NOT_TAKEN': 244, 'INSTRUMENTED_POP_JUMP_IF_TRUE': 245, 'INSTRUMENTED_POP_JUMP_IF_FALSE': 246, 'INSTRUMENTED_POP_JUMP_IF_NONE': 247, @@ -348,5 +350,5 @@ 'STORE_FAST_MAYBE_NULL': 265, } -HAVE_ARGUMENT = 41 -MIN_INSTRUMENTED_OPCODE = 237 +HAVE_ARGUMENT = 42 +MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 2dd7cf65ee3c2a..f8c4fc14c91ebe 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -29,6 +29,7 @@ def test_if_expression(self): ('LOAD_CONST', 0, 1), ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), + ('NOT_TAKEN', None, 1), ('LOAD_SMALL_INT', 42, 1), ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, @@ -49,6 +50,7 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOT_TAKEN', None, 1), ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index c719f571152d61..955a3e4cb9e4f7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -175,10 +175,11 @@ def bug708901(): %3d CALL 2 GET_ITER - L1: FOR_ITER 3 (to L2) + L1: FOR_ITER 4 (to L2) + NOT_TAKEN STORE_FAST 0 (res) -%3d JUMP_BACKWARD 5 (to L1) +%3d JUMP_BACKWARD 6 (to L1) %3d L2: END_FOR POP_TOP @@ -200,7 +201,8 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d RESUME 0 -%3d LOAD_COMMON_CONSTANT 0 (AssertionError) +%3d NOT_TAKEN + LOAD_COMMON_CONSTANT 0 (AssertionError) LOAD_CONST 0 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) @@ -432,7 +434,7 @@ def foo(a: int, b: str) -> str: 1 LOAD_SMALL_INT 0 STORE_NAME 0 (x) - 2 L1: NOP + 2 L1: NOT_TAKEN 3 LOAD_NAME 0 (x) LOAD_SMALL_INT 1 @@ -458,7 +460,8 @@ def foo(a: int, b: str) -> str: %4d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to L7) + POP_JUMP_IF_FALSE 24 (to L7) + NOT_TAKEN STORE_FAST 0 (e) %4d L4: LOAD_FAST 0 (e) @@ -555,7 +558,8 @@ def _with(c): %4d L3: PUSH_EXC_INFO WITH_EXCEPT_START TO_BOOL - POP_JUMP_IF_TRUE 1 (to L4) + POP_JUMP_IF_TRUE 2 (to L4) + NOT_TAKEN RERAISE 2 L4: POP_TOP L5: POP_EXCEPT @@ -645,7 +649,8 @@ async def _asyncwith(c): L20: CLEANUP_THROW L21: END_SEND TO_BOOL - POP_JUMP_IF_TRUE 1 (to L22) + POP_JUMP_IF_TRUE 2 (to L22) + NOT_TAKEN RERAISE 2 L22: POP_TOP L23: POP_EXCEPT @@ -839,7 +844,8 @@ def foo(x): L1: RESUME 0 LOAD_FAST 0 (.0) GET_ITER - L2: FOR_ITER 10 (to L3) + L2: FOR_ITER 11 (to L3) + NOT_TAKEN STORE_FAST 1 (z) LOAD_DEREF 2 (x) LOAD_FAST 1 (z) @@ -847,7 +853,7 @@ def foo(x): YIELD_VALUE 0 RESUME 5 POP_TOP - JUMP_BACKWARD 12 (to L2) + JUMP_BACKWARD 13 (to L2) L3: END_FOR POP_TOP LOAD_CONST 0 (None) @@ -893,14 +899,15 @@ def loop_test(): LOAD_SMALL_INT 3 BINARY_OP 5 (*) GET_ITER - L1: FOR_ITER_LIST 14 (to L2) + L1: FOR_ITER_LIST 15 (to L2) + NOT_TAKEN STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) CALL_PY_GENERAL 1 POP_TOP - JUMP_BACKWARD 16 (to L1) + JUMP_BACKWARD 17 (to L1) %3d L2: END_FOR POP_TOP @@ -1699,204 +1706,214 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=48, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=49, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=107, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=80, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_LIST', opcode=43, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_MAP', opcode=44, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=44, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=45, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=58, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=59, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=48, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=49, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=107, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=80, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=58, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=59, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=80, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=84, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=85, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=67, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=107, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=4, argval=4, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=6, argval=6, argrepr='', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=74, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=83, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=33, argval=194, argrepr='to L8', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=42, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=107, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=6, argval=6, argrepr='', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=176, argrepr='to L6', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=4, argval=4, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=192, argrepr='to L7', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=39, argval=114, argrepr='to L5', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=74, arg=11, argval=216, argrepr='to L9', offset=192, start_offset=192, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=218, start_offset=218, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=0, argval=0, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=42, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=1, argval=1, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=90, arg=1, argval=1, argrepr='__exit__', offset=232, start_offset=232, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=112, arg=2, argval=2, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=112, arg=3, argval=3, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=90, arg=0, argval=0, argrepr='__enter__', offset=238, start_offset=238, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=0, argval=0, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=107, arg=1, argval='dodgy', argrepr='dodgy', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=250, start_offset=250, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=3, argval=3, argrepr='', offset=278, start_offset=278, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=286, start_offset=286, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=288, start_offset=288, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=310, start_offset=310, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=41, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=1, argval=332, argrepr='to L11', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='RERAISE', opcode=100, arg=2, argval=2, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=73, arg=28, argval=288, argrepr='to L10', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=344, start_offset=344, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=352, start_offset=352, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=14, argval=396, argrepr='to L12', offset=364, start_offset=364, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=370, start_offset=370, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=380, start_offset=380, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=73, arg=54, argval=288, argrepr='to L10', offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=396, start_offset=396, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=398, start_offset=398, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=406, start_offset=406, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=416, start_offset=416, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=430, start_offset=430, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=68, arg=34, argval=96, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=30, start_offset=30, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=32, start_offset=32, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=52, start_offset=52, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=54, start_offset=54, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=4, argval=4, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=58, start_offset=58, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=3, argval=72, argrepr='to L2', offset=62, start_offset=62, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=66, start_offset=66, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=24, argval=24, argrepr='to L1', offset=68, start_offset=68, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=72, start_offset=72, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=6, argval=6, argrepr='', offset=74, start_offset=74, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=3, argval=90, argrepr='to L3', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=33, argval=24, argrepr='to L1', offset=86, start_offset=86, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=True, line_number=None, label=3, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=75, arg=13, argval=122, argrepr='to L5', offset=94, start_offset=94, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=98, start_offset=98, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=100, start_offset=100, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=84, arg=0, argval='i', argrepr='i', offset=122, start_offset=122, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=38, arg=None, argval=None, argrepr='', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=37, argval=210, argrepr='to L8', offset=132, start_offset=132, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=136, start_offset=136, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=138, start_offset=138, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=150, start_offset=150, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=162, start_offset=162, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=43, arg=23, argval=23, argrepr='-=', offset=164, start_offset=164, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=168, start_offset=168, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=170, start_offset=170, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=6, argval=6, argrepr='', offset=172, start_offset=172, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=174, start_offset=174, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=3, argval=188, argrepr='to L6', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=182, start_offset=182, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=33, argval=122, argrepr='to L5', offset=184, start_offset=184, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=188, start_offset=188, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=4, argval=4, argrepr='', offset=190, start_offset=190, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=192, start_offset=192, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=3, argval=206, argrepr='to L7', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=42, argval=122, argrepr='to L5', offset=202, start_offset=202, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=206, start_offset=206, starts_line=True, line_number=None, label=7, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=75, arg=11, argval=232, argrepr='to L9', offset=208, start_offset=208, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=210, start_offset=210, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=222, start_offset=222, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=0, argval=0, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=43, arg=11, argval=11, argrepr='/', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=242, start_offset=242, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=244, start_offset=244, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=91, arg=1, argval=1, argrepr='__exit__', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=113, arg=2, argval=2, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=113, arg=3, argval=3, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=91, arg=0, argval=0, argrepr='__enter__', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=0, argval=0, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=108, arg=1, argval='dodgy', argrepr='dodgy', offset=264, start_offset=264, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=266, start_offset=266, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=278, start_offset=278, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=286, start_offset=286, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=3, argval=3, argrepr='', offset=294, start_offset=294, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=302, start_offset=302, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=304, start_offset=304, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=42, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=38, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=2, argval=350, argrepr='to L11', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=2, argval=2, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=29, argval=304, argrepr='to L10', offset=360, start_offset=360, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=362, start_offset=362, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=370, start_offset=370, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=15, argval=416, argrepr='to L12', offset=382, start_offset=382, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=388, start_offset=388, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=390, start_offset=390, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=400, start_offset=400, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=56, argval=304, argrepr='to L10', offset=414, start_offset=414, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=416, start_offset=416, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=418, start_offset=418, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=426, start_offset=426, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=436, start_offset=436, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=438, start_offset=438, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=450, start_offset=450, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=452, start_offset=452, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=454, start_offset=454, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 5a4bcebedf19de..087ac8d456b843 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1491,7 +1491,15 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" +class BranchRightRecorder(JumpRecorder): + event_type = E.BRANCH_RIGHT + name = "branch right" + +class BranchLeftRecorder(JumpRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" class JumpOffsetRecorder: @@ -1504,16 +1512,23 @@ def __init__(self, events, offsets=False): def __call__(self, code, from_, to): self.events.append((self.name, code.co_name, from_, to)) -class BranchOffsetRecorder(JumpOffsetRecorder): +class BranchLeftOffsetRecorder(JumpOffsetRecorder): - event_type = E.BRANCH - name = "branch" + event_type = E.BRANCH_LEFT + name = "branch left" + +class BranchRightOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder -BRANCH_OFFSET_RECORDERS = BranchOffsetRecorder, + +BRANCHES_RECORDERS = BranchLeftRecorder, BranchRightRecorder +BRANCH_OFFSET_RECORDERS = BranchLeftOffsetRecorder, BranchRightOffsetRecorder class TestBranchAndJumpEvents(CheckEvents): maxDiff = None @@ -1529,6 +1544,11 @@ def func(): x = 6 7 + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ ('branch', 'func', 2, 2), ('branch', 'func', 3, 6), @@ -1558,6 +1578,26 @@ def func(): ('line', 'func', 7), ('line', 'get_events', 11)]) + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 3, 6), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 3, 4), + ('branch right', 'func', 2, 7)]) + + self.check_events(whilefunc, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch right', 'whilefunc', 1, 3)]) + + self.check_events(func, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + ('branch left', 'func', 28, 34), + ('branch right', 'func', 46, 60), + ('branch left', 'func', 28, 34), + ('branch left', 'func', 46, 52), + ('branch right', 'func', 28, 72)]) + def test_except_star(self): class Foo: @@ -1583,8 +1623,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, '[offset=118]'), - ('branch', 'func', '[offset=122]', '[offset=126]'), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1598,8 +1638,8 @@ def func(): ('line', 'func', 5), ('line', 'meth', 1), ('return', 'meth', None), - ('jump', 'func', 5, '[offset=118]'), - ('branch', 'func', '[offset=122]', '[offset=126]'), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), ('return', 'func', None), ('line', 'get_events', 11)]) @@ -1611,8 +1651,8 @@ def foo(n=0): n += 1 return None - in_loop = ('branch', 'foo', 10, 14) - exit_loop = ('branch', 'foo', 10, 30) + in_loop = ('branch left', 'foo', 10, 16) + exit_loop = ('branch right', 'foo', 10, 32) self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ in_loop, in_loop, @@ -1852,6 +1892,10 @@ def test_local(self): code = f1.__code__ sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL, code, E.BRANCH) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.BRANCH_LEFT | E.BRANCH_RIGHT) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) sys.monitoring.set_local_events(TEST_TOOL, code, 0) @@ -2053,7 +2097,8 @@ def setUp(self): ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), ( 2, E.CALL, capi.fire_event_call, callable, 40), ( 1, E.JUMP, capi.fire_event_jump, 60), - ( 1, E.BRANCH, capi.fire_event_branch, 70), + ( 1, E.BRANCH_RIGHT, capi.fire_event_branch_right, 70), + ( 1, E.BRANCH_LEFT, capi.fire_event_branch_left, 80), ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), diff --git a/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst b/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst new file mode 100644 index 00000000000000..6cd13572ff1893 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst @@ -0,0 +1,4 @@ +Adds two new local events to sys.monitoring, ``BRANCH_LEFT`` and +``BRANCH_RIGHT``. This allows the two arms of the branch to be disabled +independently, which should hugely improve performance of branch-level +coverage tools. The old branch event, ``BRANCH`` is now deprecated. diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index 6fd4a405688f48..e475e3b5937199 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -286,7 +286,7 @@ fire_event_jump(PyObject *self, PyObject *args) } static PyObject * -fire_event_branch(PyObject *self, PyObject *args) +fire_event_branch_right(PyObject *self, PyObject *args) { PyObject *codelike; int offset; @@ -299,7 +299,25 @@ fire_event_branch(PyObject *self, PyObject *args) if (state == NULL) { return NULL; } - int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset); + int res = PyMonitoring_FireBranchRightEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch_left(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchLeftEvent(state, codelike, offset, target_offset); RETURN_INT(teardown_fire(res, state, exception)); } @@ -478,7 +496,8 @@ static PyMethodDef TestMethods[] = { {"fire_event_call", fire_event_call, METH_VARARGS}, {"fire_event_line", fire_event_line, METH_VARARGS}, {"fire_event_jump", fire_event_jump, METH_VARARGS}, - {"fire_event_branch", fire_event_branch, METH_VARARGS}, + {"fire_event_branch_left", fire_event_branch_left, METH_VARARGS}, + {"fire_event_branch_right", fire_event_branch_right, METH_VARARGS}, {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, {"fire_event_raise", fire_event_raise, METH_VARARGS}, {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index eb8de136ee6432..ae232cae86799b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2197,6 +2197,12 @@ code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args)) return (PyObject *)new_linesiterator(code); } +static PyObject * +code_branchesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) +{ + return _PyInstrumentation_BranchesIterator(code); +} + /*[clinic input] @text_signature "($self, /, **changes)" code.replace @@ -2337,6 +2343,7 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg) static struct PyMethodDef code_methods[] = { {"__sizeof__", code_sizeof, METH_NOARGS}, {"co_lines", code_linesiterator, METH_NOARGS}, + {"co_branches", (PyCFunction)code_branchesiterator, METH_NOARGS}, {"co_positions", code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 99b0fa48e01c8b..a0007830e8cbc0 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,37 +1,37 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, - 0,0,0,0,0,243,168,0,0,0,149,0,89,0,79,0, - 70,0,111,0,89,0,79,0,70,1,111,1,88,2,31,0, - 79,1,49,1,0,0,0,0,0,0,29,0,88,2,31,0, - 79,2,88,0,77,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,49,2,0,0,0,0,0,0, - 29,0,88,1,77,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,31,0,49,0,0,0,0,0, - 0,0,79,3,2,0,0,0,111,5,79,4,16,0,67,20, - 0,0,111,6,88,2,31,0,79,5,88,6,12,0,79,6, - 88,5,88,6,2,0,0,0,12,0,47,4,49,1,0,0, - 0,0,0,0,29,0,72,22,0,0,9,0,29,0,79,0, - 33,0,41,7,78,122,18,70,114,111,122,101,110,32,72,101, - 108,108,111,32,87,111,114,108,100,122,8,115,121,115,46,97, - 114,103,118,218,6,99,111,110,102,105,103,41,5,218,12,112, - 114,111,103,114,97,109,95,110,97,109,101,218,10,101,120,101, - 99,117,116,97,98,108,101,218,15,117,115,101,95,101,110,118, - 105,114,111,110,109,101,110,116,218,17,99,111,110,102,105,103, - 117,114,101,95,99,95,115,116,100,105,111,218,14,98,117,102, - 102,101,114,101,100,95,115,116,100,105,111,122,7,99,111,110, - 102,105,103,32,122,2,58,32,41,7,218,3,115,121,115,218, - 17,95,116,101,115,116,105,110,116,101,114,110,97,108,99,97, - 112,105,218,5,112,114,105,110,116,218,4,97,114,103,118,218, - 11,103,101,116,95,99,111,110,102,105,103,115,114,2,0,0, - 0,218,3,107,101,121,169,0,243,0,0,0,0,218,18,116, - 101,115,116,95,102,114,111,122,101,110,109,97,105,110,46,112, - 121,218,8,60,109,111,100,117,108,101,62,114,17,0,0,0, - 1,0,0,0,115,94,0,0,0,240,3,1,1,1,243,8, - 0,1,11,219,0,24,225,0,5,208,6,26,212,0,27,217, - 0,5,128,106,144,35,151,40,145,40,212,0,27,216,9,26, - 215,9,38,210,9,38,211,9,40,168,24,209,9,50,128,6, - 243,2,6,12,2,128,67,241,14,0,5,10,136,71,144,67, - 144,53,152,2,152,54,160,35,153,59,152,45,208,10,40,214, - 4,41,243,15,6,12,2,114,15,0,0,0, + 0,0,0,0,0,243,170,0,0,0,149,0,90,0,80,0, + 71,0,112,0,90,0,80,0,71,1,112,1,89,2,32,0, + 80,1,50,1,0,0,0,0,0,0,30,0,89,2,32,0, + 80,2,89,0,78,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,50,2,0,0,0,0,0,0, + 30,0,89,1,78,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,32,0,50,0,0,0,0,0, + 0,0,80,3,2,0,0,0,112,5,80,4,16,0,68,21, + 0,0,28,0,112,6,89,2,32,0,80,5,89,6,12,0, + 80,6,89,5,89,6,2,0,0,0,12,0,48,4,50,1, + 0,0,0,0,0,0,30,0,73,23,0,0,9,0,30,0, + 80,0,34,0,41,7,78,122,18,70,114,111,122,101,110,32, + 72,101,108,108,111,32,87,111,114,108,100,122,8,115,121,115, + 46,97,114,103,118,218,6,99,111,110,102,105,103,41,5,218, + 12,112,114,111,103,114,97,109,95,110,97,109,101,218,10,101, + 120,101,99,117,116,97,98,108,101,218,15,117,115,101,95,101, + 110,118,105,114,111,110,109,101,110,116,218,17,99,111,110,102, + 105,103,117,114,101,95,99,95,115,116,100,105,111,218,14,98, + 117,102,102,101,114,101,100,95,115,116,100,105,111,122,7,99, + 111,110,102,105,103,32,122,2,58,32,41,7,218,3,115,121, + 115,218,17,95,116,101,115,116,105,110,116,101,114,110,97,108, + 99,97,112,105,218,5,112,114,105,110,116,218,4,97,114,103, + 118,218,11,103,101,116,95,99,111,110,102,105,103,115,114,2, + 0,0,0,218,3,107,101,121,169,0,243,0,0,0,0,218, + 18,116,101,115,116,95,102,114,111,122,101,110,109,97,105,110, + 46,112,121,218,8,60,109,111,100,117,108,101,62,114,17,0, + 0,0,1,0,0,0,115,94,0,0,0,240,3,1,1,1, + 243,8,0,1,11,219,0,24,225,0,5,208,6,26,212,0, + 27,217,0,5,128,106,144,35,151,40,145,40,212,0,27,216, + 9,26,215,9,38,210,9,38,211,9,40,168,24,209,9,50, + 128,6,244,2,6,12,2,128,67,241,14,0,5,10,136,71, + 144,67,144,53,152,2,152,54,160,35,153,59,152,45,208,10, + 40,214,4,41,243,15,6,12,2,114,15,0,0,0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b67264f0440869..cf089c368b651c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -148,6 +148,8 @@ dummy_func( RESUME_CHECK, }; + macro(NOT_TAKEN) = NOP; + op(_CHECK_PERIODIC, (--)) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); @@ -2723,7 +2725,7 @@ dummy_func( int flag = PyStackRef_IsFalse(cond); DEAD(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { @@ -2731,7 +2733,7 @@ dummy_func( int flag = PyStackRef_IsTrue(cond); DEAD(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } op(_IS_NONE, (value -- b)) { @@ -2923,13 +2925,11 @@ dummy_func( macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { - _Py_CODEUNIT *target; _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); - target = next_instr; } else { if (_PyErr_Occurred(tstate)) { @@ -2946,9 +2946,9 @@ dummy_func( STACK_SHRINK(1); PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ - target = next_instr + oparg + 2; + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); } op(_ITER_CHECK_LIST, (iter -- iter)) { @@ -4736,6 +4736,10 @@ dummy_func( INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); } + inst(INSTRUMENTED_NOT_TAKEN, ( -- )) { + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + macro(INSTRUMENTED_JUMP_BACKWARD) = unused/1 + _CHECK_PERIODIC + @@ -4744,51 +4748,43 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { _PyStackRef value_stackref = POP(); - int flag = PyStackRef_IsNone(value_stackref); - int offset; - if (flag) { - offset = oparg; + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { PyStackRef_CLOSE(value_stackref); - offset = 0; } - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { _PyStackRef value_stackref = POP(); - int offset; - int nflag = PyStackRef_IsNone(value_stackref); - if (nflag) { - offset = 0; - } - else { + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { PyStackRef_CLOSE(value_stackref); - offset = oparg; + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } tier1 inst(EXTENDED_ARG, ( -- )) { diff --git a/Python/codegen.c b/Python/codegen.c index a5e550cf8c947e..6d3272edfdbf94 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -406,7 +406,13 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_JUMP_TARGET_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); + if (_PyInstructionSequence_Addop(seq, opcode, target.id, loc) != SUCCESS) { + return ERROR; + } + if (IS_CONDITIONAL_JUMP_OPCODE(opcode) || opcode == FOR_ITER) { + return _PyInstructionSequence_Addop(seq, NOT_TAKEN, 0, NO_LOCATION); + } + return SUCCESS; } #define ADDOP_JUMP(C, LOC, OP, O) \ @@ -682,7 +688,6 @@ codegen_setup_annotations_scope(compiler *c, location loc, ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); - ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); ADDOP_I(c, loc, RAISE_VARARGS, 1); USE_LABEL(c, body); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index de61a64a6e3374..9bfc3e7f0b112b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5668,6 +5668,8 @@ /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 because it is instrumented */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ diff --git a/Python/flowgraph.c b/Python/flowgraph.c index b1097b64469ecd..64df6290de06ba 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -557,6 +557,12 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } + assert(b->b_next->b_iused > 0); + assert(b->b_next->b_instr[0].i_opcode == NOT_TAKEN); + b->b_next->b_instr[0].i_opcode = NOP; + b->b_next->b_instr[0].i_loc = NO_LOCATION; + RETURN_IF_ERROR( + basicblock_addop(backwards_jump, NOT_TAKEN, 0, last->i_loc)); RETURN_IF_ERROR( basicblock_add_jump(backwards_jump, JUMP, target, last->i_loc)); last->i_opcode = reversed_opcode; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8a89ba890fd9c9..ac89891df80590 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4618,7 +4618,6 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ - _Py_CODEUNIT *target; _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4626,7 +4625,6 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); - target = next_instr; } else { if (_PyErr_Occurred(tstate)) { @@ -4647,9 +4645,9 @@ STACK_SHRINK(1); PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ - target = next_instr + oparg + 2; + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4754,6 +4752,15 @@ GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } + TARGET(INSTRUMENTED_NOT_TAKEN) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + } + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4762,10 +4769,11 @@ /* Skip 1 cache entry */ _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); } @@ -4776,17 +4784,14 @@ INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); - int flag = PyStackRef_IsNone(value_stackref); - int offset; - if (flag) { - offset = oparg; + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { PyStackRef_CLOSE(value_stackref); - offset = 0; } - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4797,19 +4802,12 @@ INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); - int offset; - int nflag = PyStackRef_IsNone(value_stackref); - if (nflag) { - offset = 0; - } - else { + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { PyStackRef_CLOSE(value_stackref); - offset = oparg; + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4821,10 +4819,11 @@ /* Skip 1 cache entry */ _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); } @@ -6659,6 +6658,13 @@ DISPATCH(); } + TARGET(NOT_TAKEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + } + TARGET(POP_EXCEPT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -6687,7 +6693,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6719,7 +6725,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6752,7 +6758,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6770,7 +6776,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3503809e3306cb..e4255bfad8c41a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -85,22 +85,24 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, + [NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT, + [INSTRUMENTED_NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -120,6 +122,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -155,6 +158,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, @@ -323,33 +328,8 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - int opcode = - FT_ATOMIC_LOAD_UINT8_RELAXED(_PyCode_CODE(code)[offset].op.code); - assert(opcode != 0); - assert(opcode != RESERVED); - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - opcode = deinstrumented; - } - else { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(opcode != 0); - if (opcode == ENTER_EXECUTOR) { - int exec_index = _PyCode_CODE(code)[offset].op.arg; - _PyExecutorObject *exec = code->co_executors->executors[exec_index]; - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; - } - assert(!is_instrumented(opcode)); - assert(opcode != ENTER_EXECUTOR); - assert(opcode == _PyOpcode_Deopt[opcode]); - return 1 + _PyOpcode_Caches[opcode]; + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, offset); + return 1 + _PyOpcode_Caches[inst.op.code]; } #ifdef INSTRUMENT_DEBUG @@ -599,16 +579,15 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) int opcode = inst.op.code; if (opcode < MIN_INSTRUMENTED_OPCODE) { inst.op.code = _PyOpcode_Deopt[opcode]; - assert(inst.op.code <= RESUME); + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); return inst; } if (opcode == ENTER_EXECUTOR) { _PyExecutorObject *exec = code->co_executors->executors[inst.op.arg]; opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; inst.op.code = opcode; - assert(opcode <= RESUME); inst.op.arg = exec->vm_data.oparg; - assert(inst.op.code <= RESUME); + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); return inst; } if (opcode == INSTRUMENTED_LINE) { @@ -1084,6 +1063,8 @@ static const char *const event_names [] = { [PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION", [PY_MONITORING_EVENT_JUMP] = "JUMP", [PY_MONITORING_EVENT_BRANCH] = "BRANCH", + [PY_MONITORING_EVENT_BRANCH_LEFT] = "BRANCH_LEFT", + [PY_MONITORING_EVENT_BRANCH_RIGHT] = "BRANCH_RIGHT", [PY_MONITORING_EVENT_C_RETURN] = "C_RETURN", [PY_MONITORING_EVENT_PY_THROW] = "PY_THROW", [PY_MONITORING_EVENT_RAISE] = "RAISE", @@ -1111,6 +1092,10 @@ call_instrumentation_vector( /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + if (event == PY_MONITORING_EVENT_BRANCH_LEFT) { + assert(EVENT_FOR_OPCODE[_Py_GetBaseCodeUnit(code, offset-2).op.code] == PY_MONITORING_EVENT_BRANCH_RIGHT); + bytes_offset -= 4; + } PyObject *offset_obj = PyLong_FromLong(bytes_offset); if (offset_obj == NULL) { return -1; @@ -1191,7 +1176,8 @@ _Py_call_instrumentation_jump( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) { assert(event == PY_MONITORING_EVENT_JUMP || - event == PY_MONITORING_EVENT_BRANCH); + event == PY_MONITORING_EVENT_BRANCH_RIGHT || + event == PY_MONITORING_EVENT_BRANCH_LEFT); assert(frame->instr_ptr == instr); int to = (int)(target - _PyFrame_GetBytecode(frame)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1427,19 +1413,6 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* return next_opcode; } - -PyObject * -_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) -{ - PyInterpreterState *is = _PyInterpreterState_GET(); - assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); - PyObject *callback = _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], - Py_XNewRef(obj)); - - return callback; -} - static void initialize_tools(PyCodeObject *code) { @@ -2312,6 +2285,10 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) { + event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); + event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); + } if (_PyMonitoring_SetEvents(tool_id, event_set)) { return NULL; } @@ -2384,6 +2361,10 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) { + event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); + event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); + } if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); return NULL; @@ -2711,7 +2692,27 @@ _PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int3 assert(state->active); PyObject *args[4] = { NULL, NULL, NULL, target_offset }; return capi_call_instrumentation(state, codelike, offset, args, 3, - PY_MONITORING_EVENT_BRANCH); + PY_MONITORING_EVENT_BRANCH_RIGHT); +} + +int +_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_RIGHT); +} + +int +_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_LEFT); } int @@ -2849,3 +2850,213 @@ _PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelik Py_DECREF(exc); return exception_event_teardown(err, NULL); } + + + +/* Handle legacy BRANCH event */ + +typedef struct _PyLegacyBranchEventHandler { + PyObject_HEAD + vectorcallfunc vectorcall; + PyObject *handler; + bool right; + int tool_id; +} _PyLegacyBranchEventHandler; + +static void +dealloc_branch_handler(_PyLegacyBranchEventHandler *self) +{ + Py_CLEAR(self->handler); + PyObject_Free((PyObject *)self); +} + +static PyTypeObject _PyLegacyBranchEventHandler_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "sys.monitoring.branch_event_handler", + sizeof(_PyLegacyBranchEventHandler), + .tp_dealloc = (destructor)dealloc_branch_handler, + .tp_vectorcall_offset = offsetof(_PyLegacyBranchEventHandler, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_call = PyVectorcall_Call, +}; + + +static PyObject * +branch_handler( + _PyLegacyBranchEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + PyObject *res = PyObject_Vectorcall(self->handler, args, nargsf, kwnames); + if (res == &_PyInstrumentation_DISABLE) { + // Find the other instrumented instruction and remove tool + assert(PyVectorcall_NARGS(nargsf) >= 2); + PyObject *offset_obj = args[1]; + int bytes_offset = PyLong_AsLong(offset_obj); + if (PyErr_Occurred()) { + return NULL; + } + PyCodeObject *code = (PyCodeObject *)args[0]; + if (!PyCode_Check(code) || (bytes_offset & 1)) { + return res; + } + int offset = bytes_offset / 2; + /* We need FOR_ITER and POP_JUMP_ to be the same size */ + assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1); + if (self->right) { + offset += 2; + } + if (offset >= Py_SIZE(code)) { + return res; + } + int other_event = self->right ? + PY_MONITORING_EVENT_BRANCH_LEFT : PY_MONITORING_EVENT_BRANCH_RIGHT; + LOCK_CODE(code); + remove_tools(code, offset, other_event, 1 << self->tool_id); + UNLOCK_CODE(); + } + return res; +} + +static PyObject *make_branch_handler(int tool_id, PyObject *handler, bool right) +{ + _PyLegacyBranchEventHandler *callback = + PyObject_NEW(_PyLegacyBranchEventHandler, &_PyLegacyBranchEventHandler_Type); + if (callback == NULL) { + return NULL; + } + callback->vectorcall = (vectorcallfunc)branch_handler; + callback->handler = Py_NewRef(handler); + callback->right = right; + callback->tool_id = tool_id; + return (PyObject *)callback; +} + +/* Consumes a reference to obj */ +static PyObject *exchange_callables(int tool_id, int event_id, PyObject *obj) +{ + PyInterpreterState *is = _PyInterpreterState_GET(); + return _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], obj); +} + +PyObject * +_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); + PyObject *res; + if (event_id == PY_MONITORING_EVENT_BRANCH) { + PyObject *left, *right; + if (obj == NULL) { + left = NULL; + right = NULL; + } + else { + right = make_branch_handler(tool_id, obj, true); + if (right == NULL) { + return NULL; + } + left = make_branch_handler(tool_id, obj, false); + if (left == NULL) { + Py_DECREF(right); + return NULL; + } + } + Py_XDECREF(exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_RIGHT, right)); + res = exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_LEFT, left); + } + else { + res = exchange_callables(tool_id, event_id, Py_XNewRef(obj)); + } + if (res != NULL && Py_TYPE(res) == &_PyLegacyBranchEventHandler_Type) { + _PyLegacyBranchEventHandler *wrapper = (_PyLegacyBranchEventHandler *)res; + res = Py_NewRef(wrapper->handler); + Py_DECREF(wrapper); + } + return res; +} + +/* Branch Iterator */ + +typedef struct { + PyObject_HEAD + PyCodeObject *bi_code; + int bi_offset; +} branchesiterator; + +static PyObject * +int_triple(int a, int b, int c) { + PyObject *obja = PyLong_FromLong(a); + PyObject *objb = NULL; + PyObject *objc = NULL; + if (obja == NULL) { + goto error; + } + objb = PyLong_FromLong(b); + if (objb == NULL) { + goto error; + } + objc = PyLong_FromLong(c); + if (objc == NULL) { + goto error; + } + PyObject *array[3] = { obja, objb, objc }; + return _PyTuple_FromArraySteal(array, 3); +error: + Py_XDECREF(obja); + Py_XDECREF(objb); + Py_XDECREF(objc); + return NULL; +} + +static PyObject * +branchesiter_next(branchesiterator *bi) +{ + int offset = bi->bi_offset; + while (offset < Py_SIZE(bi->bi_code)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(bi->bi_code, offset); + int next_offset = offset + _PyInstruction_GetLength(bi->bi_code, offset); + int event = EVENT_FOR_OPCODE[inst.op.code]; + if (event == PY_MONITORING_EVENT_BRANCH_RIGHT) { + /* Skip NOT_TAKEN */ + int not_taken = next_offset + 1; + bi->bi_offset = not_taken; + return int_triple(offset*2, not_taken*2, (next_offset + inst.op.arg)*2); + } + offset = next_offset; + } + return NULL; +} + +static void +branchesiter_dealloc(branchesiterator *bi) +{ + Py_DECREF(bi->bi_code); + PyObject_Free(bi); +} + +static PyTypeObject _PyBranchesIterator = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "line_iterator", /* tp_name */ + sizeof(branchesiterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + .tp_dealloc = (destructor)branchesiter_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_iter = PyObject_SelfIter, + .tp_iternext = (iternextfunc)branchesiter_next, + .tp_free = PyObject_Del, +}; + +PyObject * +_PyInstrumentation_BranchesIterator(PyCodeObject *code) +{ + + branchesiterator *bi = (branchesiterator *)PyType_GenericAlloc(&_PyBranchesIterator, 0); + if (bi == NULL) { + return NULL; + } + bi->bi_code = (PyCodeObject*)Py_NewRef(code); + bi->bi_offset = 0; + return (PyObject *)bi; +} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c93941dcac4abf..7f3fb9c9a63dd1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -27,6 +27,7 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_NOP, + &&TARGET_NOT_TAKEN, &&TARGET_POP_EXCEPT, &&TARGET_POP_TOP, &&TARGET_PUSH_EXC_INFO, @@ -147,7 +148,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -235,7 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, @@ -244,6 +243,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_NOT_TAKEN, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 33b34d6fa0d3f9..2c3133d7107df2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2343,6 +2343,8 @@ /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index badd7b79102310..a1ec1927eb56df 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -106,6 +106,8 @@ Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - Python/instruction_sequence.c - _PyInstructionSequence_Type - +Python/instrumentation.c - _PyLegacyBranchEventHandler_Type - +Python/instrumentation.c - _PyBranchesIterator - Python/traceback.c - PyTraceBack_Type - ##----------------------- From 1b15c89a17ca3de6b05de5379b8717e9738c51ef Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 19 Dec 2024 10:21:17 -0800 Subject: [PATCH 10/33] gh-115999: Specialize `STORE_ATTR` in free-threaded builds. (gh-127838) * Add `_PyDictKeys_StringLookupSplit` which does locking on dict keys and use in place of `_PyDictKeys_StringLookup`. * Change `_PyObject_TryGetInstanceAttribute` to use that function in the case of split keys. * Add `unicodekeys_lookup_split` helper which allows code sharing between `_Py_dict_lookup` and `_PyDictKeys_StringLookupSplit`. * Fix locking for `STORE_ATTR_INSTANCE_VALUE`. Create `_GUARD_TYPE_VERSION_AND_LOCK` uop so that object stays locked and `tp_version_tag` cannot change. * Pass `tp_version_tag` to `specialize_dict_access()`, ensuring the version we store on the cache is the correct one (in case of it changing during the specalize analysis). * Split `analyze_descriptor` into `analyze_descriptor_load` and `analyze_descriptor_store` since those don't share much logic. Add `descriptor_is_class` helper function. * In `specialize_dict_access`, double check `_PyObject_GetManagedDict()` in case we race and dict was materialized before the lock. * Avoid borrowed references in `_Py_Specialize_StoreAttr()`. * Use `specialize()` and `unspecialize()` helpers. * Add unit tests to ensure specializing happens as expected in FT builds. * Add unit tests to attempt to trigger data races (useful for running under TSAN). * Add `has_split_table` function to `_testinternalcapi`. --- Include/internal/pycore_dict.h | 1 + Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 201 +++++++-------- Include/internal/pycore_uop_metadata.h | 6 +- Lib/test/test_free_threading/test_races.py | 141 +++++++++++ Lib/test/test_opcache.py | 66 +++++ Modules/_testinternalcapi.c | 9 + Objects/dictobject.c | 66 +++-- Python/bytecodes.c | 65 +++-- Python/executor_cases.c.h | 93 +++++-- Python/generated_cases.c.h | 86 ++++--- Python/optimizer_cases.c.h | 4 + Python/specialize.c | 271 +++++++++++++-------- 13 files changed, 716 insertions(+), 297 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6e4a308226f3fe..71927006d1cd48 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -114,6 +114,7 @@ extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); +extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e09fff062b5202..5fb236836dccd9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -2141,7 +2141,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -2340,7 +2340,7 @@ _PyOpcode_macro_expansion[256] = { [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 5fc57e48f500d0..92515b4230ccb4 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -139,15 +139,16 @@ extern "C" { #define _GUARD_TOS_FLOAT 386 #define _GUARD_TOS_INT 387 #define _GUARD_TYPE_VERSION 388 +#define _GUARD_TYPE_VERSION_AND_LOCK 389 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 389 -#define _INIT_CALL_PY_EXACT_ARGS 390 -#define _INIT_CALL_PY_EXACT_ARGS_0 391 -#define _INIT_CALL_PY_EXACT_ARGS_1 392 -#define _INIT_CALL_PY_EXACT_ARGS_2 393 -#define _INIT_CALL_PY_EXACT_ARGS_3 394 -#define _INIT_CALL_PY_EXACT_ARGS_4 395 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 390 +#define _INIT_CALL_PY_EXACT_ARGS 391 +#define _INIT_CALL_PY_EXACT_ARGS_0 392 +#define _INIT_CALL_PY_EXACT_ARGS_1 393 +#define _INIT_CALL_PY_EXACT_ARGS_2 394 +#define _INIT_CALL_PY_EXACT_ARGS_3 395 +#define _INIT_CALL_PY_EXACT_ARGS_4 396 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -160,143 +161,143 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 396 -#define _IS_NONE 397 +#define _INTERNAL_INCREMENT_OPT_COUNTER 397 +#define _IS_NONE 398 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 398 -#define _ITER_CHECK_RANGE 399 -#define _ITER_CHECK_TUPLE 400 -#define _ITER_JUMP_LIST 401 -#define _ITER_JUMP_RANGE 402 -#define _ITER_JUMP_TUPLE 403 -#define _ITER_NEXT_LIST 404 -#define _ITER_NEXT_RANGE 405 -#define _ITER_NEXT_TUPLE 406 -#define _JUMP_TO_TOP 407 +#define _ITER_CHECK_LIST 399 +#define _ITER_CHECK_RANGE 400 +#define _ITER_CHECK_TUPLE 401 +#define _ITER_JUMP_LIST 402 +#define _ITER_JUMP_RANGE 403 +#define _ITER_JUMP_TUPLE 404 +#define _ITER_NEXT_LIST 405 +#define _ITER_NEXT_RANGE 406 +#define _ITER_NEXT_TUPLE 407 +#define _JUMP_TO_TOP 408 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 408 -#define _LOAD_ATTR_CLASS 409 -#define _LOAD_ATTR_CLASS_0 410 -#define _LOAD_ATTR_CLASS_1 411 +#define _LOAD_ATTR 409 +#define _LOAD_ATTR_CLASS 410 +#define _LOAD_ATTR_CLASS_0 411 +#define _LOAD_ATTR_CLASS_1 412 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 412 -#define _LOAD_ATTR_INSTANCE_VALUE_0 413 -#define _LOAD_ATTR_INSTANCE_VALUE_1 414 -#define _LOAD_ATTR_METHOD_LAZY_DICT 415 -#define _LOAD_ATTR_METHOD_NO_DICT 416 -#define _LOAD_ATTR_METHOD_WITH_VALUES 417 -#define _LOAD_ATTR_MODULE 418 -#define _LOAD_ATTR_MODULE_FROM_KEYS 419 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 420 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 421 -#define _LOAD_ATTR_PROPERTY_FRAME 422 -#define _LOAD_ATTR_SLOT 423 -#define _LOAD_ATTR_SLOT_0 424 -#define _LOAD_ATTR_SLOT_1 425 -#define _LOAD_ATTR_WITH_HINT 426 +#define _LOAD_ATTR_INSTANCE_VALUE 413 +#define _LOAD_ATTR_INSTANCE_VALUE_0 414 +#define _LOAD_ATTR_INSTANCE_VALUE_1 415 +#define _LOAD_ATTR_METHOD_LAZY_DICT 416 +#define _LOAD_ATTR_METHOD_NO_DICT 417 +#define _LOAD_ATTR_METHOD_WITH_VALUES 418 +#define _LOAD_ATTR_MODULE 419 +#define _LOAD_ATTR_MODULE_FROM_KEYS 420 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 421 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 422 +#define _LOAD_ATTR_PROPERTY_FRAME 423 +#define _LOAD_ATTR_SLOT 424 +#define _LOAD_ATTR_SLOT_0 425 +#define _LOAD_ATTR_SLOT_1 426 +#define _LOAD_ATTR_WITH_HINT 427 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 427 +#define _LOAD_BYTECODE 428 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 428 -#define _LOAD_CONST_INLINE_BORROW 429 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 430 -#define _LOAD_CONST_INLINE_WITH_NULL 431 +#define _LOAD_CONST_INLINE 429 +#define _LOAD_CONST_INLINE_BORROW 430 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431 +#define _LOAD_CONST_INLINE_WITH_NULL 432 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 432 -#define _LOAD_FAST_0 433 -#define _LOAD_FAST_1 434 -#define _LOAD_FAST_2 435 -#define _LOAD_FAST_3 436 -#define _LOAD_FAST_4 437 -#define _LOAD_FAST_5 438 -#define _LOAD_FAST_6 439 -#define _LOAD_FAST_7 440 +#define _LOAD_FAST 433 +#define _LOAD_FAST_0 434 +#define _LOAD_FAST_1 435 +#define _LOAD_FAST_2 436 +#define _LOAD_FAST_3 437 +#define _LOAD_FAST_4 438 +#define _LOAD_FAST_5 439 +#define _LOAD_FAST_6 440 +#define _LOAD_FAST_7 441 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 441 -#define _LOAD_GLOBAL_BUILTINS 442 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 443 -#define _LOAD_GLOBAL_MODULE 444 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 445 +#define _LOAD_GLOBAL 442 +#define _LOAD_GLOBAL_BUILTINS 443 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 444 +#define _LOAD_GLOBAL_MODULE 445 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 446 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 446 -#define _LOAD_SMALL_INT_0 447 -#define _LOAD_SMALL_INT_1 448 -#define _LOAD_SMALL_INT_2 449 -#define _LOAD_SMALL_INT_3 450 +#define _LOAD_SMALL_INT 447 +#define _LOAD_SMALL_INT_0 448 +#define _LOAD_SMALL_INT_1 449 +#define _LOAD_SMALL_INT_2 450 +#define _LOAD_SMALL_INT_3 451 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 451 +#define _MAKE_CALLARGS_A_TUPLE 452 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 452 +#define _MAKE_WARM 453 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 453 -#define _MAYBE_EXPAND_METHOD_KW 454 -#define _MONITOR_CALL 455 -#define _MONITOR_JUMP_BACKWARD 456 -#define _MONITOR_RESUME 457 +#define _MAYBE_EXPAND_METHOD 454 +#define _MAYBE_EXPAND_METHOD_KW 455 +#define _MONITOR_CALL 456 +#define _MONITOR_JUMP_BACKWARD 457 +#define _MONITOR_RESUME 458 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 458 -#define _POP_JUMP_IF_TRUE 459 +#define _POP_JUMP_IF_FALSE 459 +#define _POP_JUMP_IF_TRUE 460 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 461 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 461 +#define _PUSH_FRAME 462 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 462 -#define _PY_FRAME_KW 463 -#define _QUICKEN_RESUME 464 -#define _REPLACE_WITH_TRUE 465 +#define _PY_FRAME_GENERAL 463 +#define _PY_FRAME_KW 464 +#define _QUICKEN_RESUME 465 +#define _REPLACE_WITH_TRUE 466 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 466 -#define _SEND 467 -#define _SEND_GEN_FRAME 468 +#define _SAVE_RETURN_OFFSET 467 +#define _SEND 468 +#define _SEND_GEN_FRAME 469 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 469 -#define _STORE_ATTR 470 -#define _STORE_ATTR_INSTANCE_VALUE 471 -#define _STORE_ATTR_SLOT 472 -#define _STORE_ATTR_WITH_HINT 473 +#define _START_EXECUTOR 470 +#define _STORE_ATTR 471 +#define _STORE_ATTR_INSTANCE_VALUE 472 +#define _STORE_ATTR_SLOT 473 +#define _STORE_ATTR_WITH_HINT 474 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 474 -#define _STORE_FAST_0 475 -#define _STORE_FAST_1 476 -#define _STORE_FAST_2 477 -#define _STORE_FAST_3 478 -#define _STORE_FAST_4 479 -#define _STORE_FAST_5 480 -#define _STORE_FAST_6 481 -#define _STORE_FAST_7 482 +#define _STORE_FAST 475 +#define _STORE_FAST_0 476 +#define _STORE_FAST_1 477 +#define _STORE_FAST_2 478 +#define _STORE_FAST_3 479 +#define _STORE_FAST_4 480 +#define _STORE_FAST_5 481 +#define _STORE_FAST_6 482 +#define _STORE_FAST_7 483 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 483 -#define _STORE_SUBSCR 484 +#define _STORE_SLICE 484 +#define _STORE_SUBSCR 485 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 485 -#define _TO_BOOL 486 +#define _TIER2_RESUME_CHECK 486 +#define _TO_BOOL 487 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -306,13 +307,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 487 +#define _UNPACK_SEQUENCE 488 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 487 +#define MAX_UOP_ID 488 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index eadfda472a7270..e71194b116e020 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -148,6 +148,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, + [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, @@ -167,7 +168,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_ATTR_SLOT] = 0, + [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -428,6 +429,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", [_IMPORT_FROM] = "_IMPORT_FROM", [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", @@ -839,6 +841,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_TYPE_VERSION: return 0; + case _GUARD_TYPE_VERSION_AND_LOCK: + return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 0; case _LOAD_ATTR_INSTANCE_VALUE_0: diff --git a/Lib/test/test_free_threading/test_races.py b/Lib/test/test_free_threading/test_races.py index 09e1d52e3509f9..69982558a067a5 100644 --- a/Lib/test/test_free_threading/test_races.py +++ b/Lib/test/test_free_threading/test_races.py @@ -4,6 +4,7 @@ import threading import time import unittest +import _testinternalcapi from test.support import threading_helper @@ -129,6 +130,146 @@ def mutate(): # with the cell binding being changed). do_race(access, mutate) + def test_racing_to_bool(self): + + seq = [1] + + class C: + def __bool__(self): + return False + + def access(): + if seq: + return 1 + else: + return 2 + + def mutate(): + nonlocal seq + seq = [1] + time.sleep(0) + seq = C() + time.sleep(0) + + do_race(access, mutate) + + def test_racing_store_attr_slot(self): + class C: + __slots__ = ['x', '__dict__'] + + c = C() + + def set_slot(): + for i in range(10): + c.x = i + time.sleep(0) + + def change_type(): + def set_x(self, x): + pass + + def get_x(self): + pass + + C.x = property(get_x, set_x) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(set_slot, change_type) + + def set_getattribute(): + C.__getattribute__ = lambda self, x: x + time.sleep(0) + del C.__getattribute__ + time.sleep(0) + + do_race(set_slot, set_getattribute) + + def test_racing_store_attr_instance_value(self): + class C: + pass + + c = C() + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + def read(): + x = c.x + + def mutate(): + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(read, mutate) + + def test_racing_store_attr_with_hint(self): + class C: + pass + + c = C() + for i in range(29): + setattr(c, f"_{i}", None) + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + def read(): + x = c.x + + def mutate(): + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(read, mutate) + + def make_shared_key_dict(self): + class C: + pass + + a = C() + a.x = 1 + return a.__dict__ + + def test_racing_store_attr_dict(self): + """Test STORE_ATTR with various dictionary types.""" + class C: + pass + + c = C() + + def set_value(): + for i in range(20): + c.x = i + + def mutate(): + nonlocal c + c.x = 1 + self.assertTrue(_testinternalcapi.has_inline_values(c)) + for i in range(30): + setattr(c, f"_{i}", None) + self.assertFalse(_testinternalcapi.has_inline_values(c.__dict__)) + c.__dict__ = self.make_shared_key_dict() + self.assertTrue(_testinternalcapi.has_split_table(c.__dict__)) + c.__dict__[1] = None + self.assertFalse(_testinternalcapi.has_split_table(c.__dict__)) + c = C() + + do_race(set_value, mutate) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 94709e2022550a..ad0b0c487a4118 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1383,6 +1383,72 @@ def send_yield_from(): self.assert_specialized(send_yield_from, "SEND_GEN") self.assert_no_opcode(send_yield_from, "SEND") + @cpython_only + @requires_specialization_ft + def test_store_attr_slot(self): + class C: + __slots__ = ['x'] + + def set_slot(): + c = C() + for i in range(100): + c.x = i + + set_slot() + + self.assert_specialized(set_slot, "STORE_ATTR_SLOT") + self.assert_no_opcode(set_slot, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_slot() + self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT") + + @cpython_only + @requires_specialization_ft + def test_store_attr_instance_value(self): + class C: + pass + + def set_value(): + c = C() + for i in range(100): + c.x = i + + set_value() + + self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value() + self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE") + + @cpython_only + @requires_specialization_ft + def test_store_attr_with_hint(self): + class C: + pass + + c = C() + for i in range(29): + setattr(c, f"_{i}", None) + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value() + self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT") + @cpython_only @requires_specialization_ft def test_to_bool(self): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 014f89997f7f60..150d34d168f5e4 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1989,6 +1989,14 @@ has_inline_values(PyObject *self, PyObject *obj) Py_RETURN_FALSE; } +static PyObject * +has_split_table(PyObject *self, PyObject *obj) +{ + if (PyDict_Check(obj) && _PyDict_HasSplitTable((PyDictObject *)obj)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} // Circumvents standard version assignment machinery - use with caution and only on // short-lived heap types @@ -2139,6 +2147,7 @@ static PyMethodDef module_functions[] = { {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, {"has_inline_values", has_inline_values, METH_O}, + {"has_split_table", has_split_table, METH_O}, {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, PyDoc_STR("forcefully assign type->tp_version_tag")}, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 05c93a3e448181..2a054c3f2ae0ff 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1129,6 +1129,35 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P return do_lookup(mp, dk, key, hash, compare_generic); } +#ifdef Py_GIL_DISABLED +static Py_ssize_t +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, + Py_hash_t hash); +#endif + +static Py_ssize_t +unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + Py_ssize_t ix; + assert(dk->dk_kind == DICT_KEYS_SPLIT); + assert(PyUnicode_CheckExact(key)); + +#ifdef Py_GIL_DISABLED + // A split dictionaries keys can be mutated by other dictionaries + // but if we have a unicode key we can avoid locking the shared + // keys. + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + UNLOCK_KEYS(dk); + } +#else + ix = unicodekeys_lookup_unicode(dk, key, hash); +#endif + return ix; +} + /* Lookup a string in a (all unicode) dict keys. * Returns DKIX_ERROR if key is not a string, * or if the dict keys is not all strings. @@ -1153,13 +1182,24 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) return unicodekeys_lookup_unicode(dk, key, hash); } -#ifdef Py_GIL_DISABLED - -static Py_ssize_t -unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, - Py_hash_t hash); - -#endif +/* Like _PyDictKeys_StringLookup() but only works on split keys. Note + * that in free-threaded builds this locks the keys object as required. + */ +Py_ssize_t +_PyDictKeys_StringLookupSplit(PyDictKeysObject* dk, PyObject *key) +{ + assert(dk->dk_kind == DICT_KEYS_SPLIT); + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = unicode_get_hash(key); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(key); + if (hash == -1) { + PyErr_Clear(); + return DKIX_ERROR; + } + } + return unicodekeys_lookup_split(dk, key, hash); +} /* The basic lookup function used by all operations. @@ -1192,15 +1232,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu if (PyUnicode_CheckExact(key)) { #ifdef Py_GIL_DISABLED if (kind == DICT_KEYS_SPLIT) { - // A split dictionaries keys can be mutated by other - // dictionaries but if we have a unicode key we can avoid - // locking the shared keys. - ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); - if (ix == DKIX_KEY_CHANGED) { - LOCK_KEYS(dk); - ix = unicodekeys_lookup_unicode(dk, key, hash); - UNLOCK_KEYS(dk); - } + ix = unicodekeys_lookup_split(dk, key, hash); } else { ix = unicodekeys_lookup_unicode(dk, key, hash); @@ -6967,7 +6999,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); - Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); + Py_ssize_t ix = _PyDictKeys_StringLookupSplit(keys, name); if (ix == DKIX_EMPTY) { *attr = NULL; return true; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cf089c368b651c..30c12dd4dc9205 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1467,7 +1467,7 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -1476,7 +1476,7 @@ dummy_func( } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_STORE_ATTR, (v, owner --)) { @@ -2129,7 +2129,18 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - EXIT_IF(tp->tp_version_tag != type_version); + EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version); + } + + op(_GUARD_TYPE_VERSION_AND_LOCK, (type_version/2, owner -- owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + EXIT_IF(!LOCK_OBJECT(owner_o)); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + EXIT_IF(true); + } } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { @@ -2336,8 +2347,11 @@ dummy_func( assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - EXIT_IF(_PyObject_GetManagedDict(owner_o)); - EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + EXIT_IF(true); + } } op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) { @@ -2347,21 +2361,20 @@ dummy_func( assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + - _GUARD_TYPE_VERSION + + _GUARD_TYPE_VERSION_AND_LOCK + _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; @@ -2370,16 +2383,34 @@ dummy_func( assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); + DEOPT_IF(!LOCK_OBJECT(dict)); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } + #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } PyObject *old_value = ep->me_value; - DEOPT_IF(old_value == NULL); + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -2395,10 +2426,12 @@ dummy_func( op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o)); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9bfc3e7f0b112b..6e752c57cd70f3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2574,13 +2574,34 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - if (tp->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } break; } + case _GUARD_TYPE_VERSION_AND_LOCK: { + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + break; + } + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -2910,13 +2931,13 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (_PyObject_InlineValues(owner_o)->valid == 0) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } @@ -2932,15 +2953,14 @@ assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2961,30 +2981,50 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries) { + if (!LOCK_OBJECT(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!DK_IS_UNICODE(dict->ma_keys)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -3002,10 +3042,15 @@ value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ac89891df80590..ee5c55a832d460 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5319,7 +5319,7 @@ uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5388,7 +5388,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { @@ -5433,7 +5433,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5476,7 +5476,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5512,7 +5512,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { @@ -5611,7 +5611,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5642,7 +5642,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { @@ -5688,7 +5688,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5750,7 +5750,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5787,7 +5787,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { @@ -7314,7 +7314,7 @@ owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -7325,7 +7325,7 @@ } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 3 cache entries */ // _STORE_ATTR @@ -7353,21 +7353,29 @@ _PyStackRef owner; _PyStackRef value; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION + // _GUARD_TYPE_VERSION_AND_LOCK { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _GUARD_DORV_NO_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR); - DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _STORE_ATTR_INSTANCE_VALUE { @@ -7378,15 +7386,14 @@ assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } stack_pointer += -2; @@ -7408,17 +7415,19 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } @@ -7441,7 +7450,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7451,18 +7460,35 @@ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } PyObject *old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -7815,7 +7841,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2c3133d7107df2..b46079ec8a1992 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1121,6 +1121,10 @@ break; } + case _GUARD_TYPE_VERSION_AND_LOCK: { + break; + } + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { break; } diff --git a/Python/specialize.c b/Python/specialize.c index 6c45320f95db8e..349ed472298945 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code); #ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -#endif static uint32_t type_get_version(PyTypeObject *t, int opcode); +#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -881,71 +881,142 @@ classify_descriptor(PyObject *descriptor, bool has_getattr) return NON_DESCRIPTOR; } -static DescriptorClassification -analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) +static bool +descriptor_is_class(PyObject *descriptor, PyObject *name) { + return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) && + (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); +} + +#ifndef Py_GIL_DISABLED +static DescriptorClassification +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { bool has_getattr = false; - if (store) { - if (type->tp_setattro != PyObject_GenericSetAttr) { + getattrofunc getattro_slot = type->tp_getattro; + if (getattro_slot == PyObject_GenericGetAttr) { + /* Normal attribute lookup; */ + has_getattr = false; + } + else if (getattro_slot == _Py_slot_tp_getattr_hook || + getattro_slot == _Py_slot_tp_getattro) { + /* One or both of __getattribute__ or __getattr__ may have been + overridden See typeobject.c for why these functions are special. */ + PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + bool has_custom_getattribute = getattribute != NULL && + getattribute != interp->callable_cache.object__getattribute__; + PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + has_getattr = getattr != NULL; + Py_XDECREF(getattr); + if (has_custom_getattribute) { + if (getattro_slot == _Py_slot_tp_getattro && + !has_getattr && + Py_IS_TYPE(getattribute, &PyFunction_Type)) { + *descr = getattribute; + return GETATTRIBUTE_IS_PYTHON_FUNCTION; + } + /* Potentially both __getattr__ and __getattribute__ are set. + Too complicated */ + Py_DECREF(getattribute); *descr = NULL; return GETSET_OVERRIDDEN; } + /* Potentially has __getattr__ but no custom __getattribute__. + Fall through to usual descriptor analysis. + Usual attribute lookup should only be allowed at runtime + if we can guarantee that there is no way an exception can be + raised. This means some specializations, e.g. specializing + for property() isn't safe. + */ + Py_XDECREF(getattribute); } else { - getattrofunc getattro_slot = type->tp_getattro; - if (getattro_slot == PyObject_GenericGetAttr) { - /* Normal attribute lookup; */ - has_getattr = false; - } - else if (getattro_slot == _Py_slot_tp_getattr_hook || - getattro_slot == _Py_slot_tp_getattro) { - /* One or both of __getattribute__ or __getattr__ may have been - overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_Lookup(type, - &_Py_ID(__getattribute__)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - bool has_custom_getattribute = getattribute != NULL && - getattribute != interp->callable_cache.object__getattribute__; - has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL; - if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && - Py_IS_TYPE(getattribute, &PyFunction_Type)) { - *descr = getattribute; - return GETATTRIBUTE_IS_PYTHON_FUNCTION; - } - /* Potentially both __getattr__ and __getattribute__ are set. - Too complicated */ - *descr = NULL; - return GETSET_OVERRIDDEN; - } - /* Potentially has __getattr__ but no custom __getattribute__. - Fall through to usual descriptor analysis. - Usual attribute lookup should only be allowed at runtime - if we can guarantee that there is no way an exception can be - raised. This means some specializations, e.g. specializing - for property() isn't safe. - */ - } - else { - *descr = NULL; - return GETSET_OVERRIDDEN; - } + *descr = NULL; + return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_Lookup(type, name); + PyObject *descriptor = _PyType_LookupRef(type, name); *descr = descriptor; - if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { - if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { - return DUNDER_CLASS; - } + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } +#endif //!Py_GIL_DISABLED + +static DescriptorClassification +analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) +{ + if (type->tp_setattro != PyObject_GenericSetAttr) { + *descr = NULL; + return GETSET_OVERRIDDEN; + } + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version); + *descr = descriptor; + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; + } + return classify_descriptor(descriptor, false); +} + +static int +specialize_dict_access_inline( + PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int values_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(PyUnicode_CheckExact(name)); + Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); + assert (index != DKIX_ERROR); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)offset; + write_u32(cache->version, tp_version); + specialize(instr, values_op); + return 1; +} + +static int +specialize_dict_access_hint( + PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int hint_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + // We found an instance with a __dict__. + if (_PyDict_HasSplitTable(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); + return 0; + } + Py_ssize_t index = _PyDict_LookupIndex(dict, name); + if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)index; + write_u32(cache->version, tp_version); + specialize(instr, hint_op); + return 1; +} + static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || @@ -956,29 +1027,25 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && - _PyObject_InlineValues(owner)->valid && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) && !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - assert(PyUnicode_CheckExact(name)); - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - assert (index != DKIX_ERROR); - if (index == DKIX_EMPTY) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); - return 0; + int res; + Py_BEGIN_CRITICAL_SECTION(owner); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + if (dict == NULL) { + // managed dict, not materialized, inline values valid + res = specialize_dict_access_inline(owner, instr, type, kind, name, + tp_version, base_op, values_op); } - assert(index >= 0); - char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; - Py_ssize_t offset = value_addr - (char *)owner; - if (offset != (uint16_t)offset) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); - return 0; + else { + // lost race and dict was created, fail specialization + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); + res = 0; } - write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)offset; - specialize(instr, values_op); + Py_END_CRITICAL_SECTION(); + return res; } else { PyDictObject *dict = _PyObject_GetManagedDict(owner); @@ -986,25 +1053,14 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } - // We found an instance with a __dict__. - if (dict->ma_values) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); - return 0; - } - Py_ssize_t index = - _PyDict_LookupIndex(dict, name); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_DICT : - SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache->index = (uint16_t)index; - write_u32(cache->version, type->tp_version_tag); - specialize(instr, hint_op); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + // materialized managed dict + res = specialize_dict_access_hint(dict, instr, type, kind, name, + tp_version, base_op, hint_op); + Py_END_CRITICAL_SECTION(); + return res; } - return 1; } #ifndef Py_GIL_DISABLED @@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na PyTypeObject *type = Py_TYPE(owner); bool shadow = instance_has_key(owner, name); PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); if (type_get_version(type, LOAD_ATTR) == 0) { return -1; @@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } @@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na { PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); + PyObject *descr = NULL; _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (!_PyType_IsReady(type)) { @@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; } - PyObject *descr; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); - if (type_get_version(type, STORE_ATTR) == 0) { + unsigned int tp_version = 0; + DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version); + if (tp_version == 0) { goto fail; } + assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = STORE_ATTR_SLOT; + write_u32(cache->version, tp_version); + specialize(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_WITH_HINT)) { goto success; } } fail: - STAT_INC(STORE_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + Py_XDECREF(descr); + unspecialize(instr); return; success: - STAT_INC(STORE_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + Py_XDECREF(descr); + return; } #ifndef Py_GIL_DISABLED @@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor(cls, name, &descr, 0); + kind = analyze_descriptor_load(cls, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref if (type_get_version(cls, LOAD_ATTR) == 0) { return -1; } @@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode) } return version; } -#endif // Py_GIL_DISABLED /* Returning 0 indicates a failure. */ static uint32_t @@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode) } return version; } +#endif // Py_GIL_DISABLED void _Py_Specialize_BinarySubscr( From e163e8d4e1a9844b8615ef38b9917b887a377948 Mon Sep 17 00:00:00 2001 From: Zhikang Yan <2951256653@qq.com> Date: Fri, 20 Dec 2024 04:24:47 +0800 Subject: [PATCH 11/33] gh-128062: Fix the font size and shortcut display of the turtledemo menu (#128063) Leave the font of the menu bar the default to keep it consistent with the rest of the world. Display the shortcut keys in the right way, using the 'accelerator' option. --------- Co-authored-by: Peter Bierma Co-authored-by: Terry Jan Reedy --- Lib/turtledemo/__main__.py | 17 +++++++---------- ...24-12-18-10-18-55.gh-issue-128062.E9oU7-.rst | 2 ++ 2 files changed, 9 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index 9c15916fb6672e..b49c0beab3ccf7 100644 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -105,7 +105,6 @@ DONE = 4 EVENTDRIVEN = 5 -menufont = ("Arial", 12, NORMAL) btnfont = ("Arial", 12, 'bold') txtfont = ['Lucida Console', 10, 'normal'] @@ -297,23 +296,21 @@ def makeLoadDemoMenu(self, master): for entry in getExampleEntries(): def load(entry=entry): self.loadfile(entry) - menu.add_command(label=entry, underline=0, - font=menufont, command=load) + menu.add_command(label=entry, underline=0, command=load) return menu def makeFontMenu(self, master): menu = Menu(master, tearoff=0) - menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, - font=menufont) - menu.add_command(label="Increase (C-'+')", command=self.increase_size, - font=menufont) + menu.add_command(label="Decrease", command=self.decrease_size, + accelerator=f"{'Command' if darwin else 'Ctrl'}+-") + menu.add_command(label="Increase", command=self.increase_size, + accelerator=f"{'Command' if darwin else 'Ctrl'}+=") menu.add_separator() for size in font_sizes: def resize(size=size): self.set_txtsize(size) - menu.add_command(label=str(size), underline=0, - font=menufont, command=resize) + menu.add_command(label=str(size), underline=0, command=resize) return menu def makeHelpMenu(self, master): @@ -322,7 +319,7 @@ def makeHelpMenu(self, master): for help_label, help_file in help_entries: def show(help_label=help_label, help_file=help_file): view_text(self.root, help_label, help_file) - menu.add_command(label=help_label, font=menufont, command=show) + menu.add_command(label=help_label, command=show) return menu def refreshCanvas(self): diff --git a/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst b/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst new file mode 100644 index 00000000000000..d8e262e0848077 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst @@ -0,0 +1,2 @@ +Revert the font of :mod:`turtledemo`'s menu bar to its default value and +display the shortcut keys in the correct position. From 255762c09fe518757bb3e8ce1bb6e5d8eec9f466 Mon Sep 17 00:00:00 2001 From: mpage Date: Thu, 19 Dec 2024 13:03:14 -0800 Subject: [PATCH 12/33] gh-127274: Defer nested methods (#128012) Methods (functions defined in class scope) are likely to be cleaned up by the GC anyway. Add a new code flag, `CO_METHOD`, that is set for functions defined in a class scope. Use that when deciding to defer functions. --- Doc/library/inspect.rst | 7 +++++++ Include/cpython/code.h | 3 +++ Include/internal/pycore_symtable.h | 1 + Lib/dis.py | 1 + Lib/inspect.py | 1 + Lib/test/test_monitoring.py | 9 +++------ Lib/test/test_opcache.py | 11 ++++------- .../2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst | 3 +++ Objects/funcobject.c | 6 +++++- Python/compile.c | 2 ++ Python/symtable.c | 7 +++++++ 11 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ca5dac87aff2b4..0085207d3055f2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1708,6 +1708,13 @@ which is a bitmap of the following flags: .. versionadded:: 3.14 +.. data:: CO_METHOD + + The flag is set when the code object is a function defined in class + scope. + + .. versionadded:: 3.14 + .. note:: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation diff --git a/Include/cpython/code.h b/Include/cpython/code.h index c3c0165d556ead..cb6261ddde941b 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -199,6 +199,9 @@ struct PyCodeObject _PyCode_DEF(1); */ #define CO_HAS_DOCSTRING 0x4000000 +/* A function defined in class scope */ +#define CO_METHOD 0x8000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 91dac767d5885b..b7e274296112aa 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -124,6 +124,7 @@ typedef struct _symtable_entry { unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ unsigned ste_has_docstring : 1; /* true if docstring present */ + unsigned ste_method : 1; /* true if block is a function block defined in class scope */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ diff --git a/Lib/dis.py b/Lib/dis.py index aa22404c6687e1..109c986bbe3d7d 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -162,6 +162,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets 256: "ITERABLE_COROUTINE", 512: "ASYNC_GENERATOR", 0x4000000: "HAS_DOCSTRING", + 0x8000000: "METHOD", } def pretty_flags(flags): diff --git a/Lib/inspect.py b/Lib/inspect.py index b7d8271f8a471f..5b7c4df8927c87 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -57,6 +57,7 @@ "CO_VARARGS", "CO_VARKEYWORDS", "CO_HAS_DOCSTRING", + "CO_METHOD", "ClassFoundException", "ClosureVars", "EndOfBlock", diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 087ac8d456b843..32b3a6ac049e28 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -850,12 +850,6 @@ def __init__(self, events): def __call__(self, code, offset, val): self.events.append(("return", code.co_name, val)) -# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that -# are deferred. We only defer functions defined at the top-level. -class ValueErrorRaiser: - def __init__(self): - raise ValueError() - class ExceptionMonitoringTest(CheckEvents): @@ -1054,6 +1048,9 @@ def func(): @requires_specialization_ft def test_no_unwind_for_shim_frame(self): + class ValueErrorRaiser: + def __init__(self): + raise ValueError() def f(): try: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index ad0b0c487a4118..ba111b5117b41d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -493,13 +493,6 @@ def f(): self.assertFalse(f()) -# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that -# are deferred. We only defer functions defined at the top-level. -class MyClass: - def __init__(self): - pass - - class InitTakesArg: def __init__(self, arg): self.arg = arg @@ -536,6 +529,10 @@ def f(x, y): @disabling_optimizer @requires_specialization_ft def test_assign_init_code(self): + class MyClass: + def __init__(self): + pass + def instantiate(): return MyClass() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst new file mode 100644 index 00000000000000..a4608fbbbf19ec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst @@ -0,0 +1,3 @@ +Add a new flag, ``CO_METHOD``, to :attr:`~codeobject.co_flags` that +indicates whether the code object belongs to a function defined in class +scope. diff --git a/Objects/funcobject.c b/Objects/funcobject.c index cca7f01498013e..7b17a9ba31fac4 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -210,10 +210,14 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = FUNC_VERSION_UNSET; - if ((code_obj->co_flags & CO_NESTED) == 0) { + if (((code_obj->co_flags & CO_NESTED) == 0) || + (code_obj->co_flags & CO_METHOD)) { // Use deferred reference counting for top-level functions, but not // nested functions because they are more likely to capture variables, // which makes prompt deallocation more important. + // + // Nested methods (functions defined in class scope) are also deferred, + // since they will likely be cleaned up by GC anyway. _PyObject_SetDeferredRefcount((PyObject *)op); } _PyObject_GC_TRACK(op); diff --git a/Python/compile.c b/Python/compile.c index cbfba7f493e07d..ef470830336dde 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1289,6 +1289,8 @@ compute_code_flags(compiler *c) flags |= CO_VARKEYWORDS; if (ste->ste_has_docstring) flags |= CO_HAS_DOCSTRING; + if (ste->ste_method) + flags |= CO_METHOD; } if (ste->ste_coroutine && !ste->ste_generator) { diff --git a/Python/symtable.c b/Python/symtable.c index ebddb0b93fca0a..49bd01ba68ac9e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -138,6 +138,13 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_has_docstring = 0; + ste->ste_method = 0; + if (st->st_cur != NULL && + st->st_cur->ste_type == ClassBlock && + block == FunctionBlock) { + ste->ste_method = 1; + } + ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); ste->ste_children = PyList_New(0); From c14db202750ff9eaf3919298f1172270b7dfd64e Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Thu, 19 Dec 2024 14:07:17 -0800 Subject: [PATCH 13/33] gh-128080: remove unnecessary `__init__` method from Enum (GH-128081) remove unnecessary __init__ method from Enum Co-authored-by: Peter Bierma --- Lib/enum.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 27be3fb83b2afb..ccc1da42206474 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1211,9 +1211,6 @@ def __new__(cls, value): exc = None ve_exc = None - def __init__(self, *args, **kwds): - pass - def _add_alias_(self, name): self.__class__._add_member_(name, self) From 39e69a7cd54d44c9061db89bb15c460d30fba7a6 Mon Sep 17 00:00:00 2001 From: Md Rokibul Islam Date: Fri, 20 Dec 2024 00:38:42 +0100 Subject: [PATCH 14/33] gh-112328: Document EnumDict in docs and release notes (GH-121720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Ethan Furman --- Doc/library/enum.rst | 17 ++++++++++++++++- Doc/whatsnew/3.13.rst | 5 +++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 16a9b0326e9f3d..51292a11f507c4 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -149,9 +149,14 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :class:`EnumDict` + + A subclass of :class:`dict` for use when subclassing :class:`EnumType`. + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` +.. versionadded:: 3.14 ``EnumDict`` --------------- @@ -821,7 +826,17 @@ Data Types >>> KeepFlag(2**2 + 2**4) -.. versionadded:: 3.11 + .. versionadded:: 3.11 + +.. class:: EnumDict + + *EnumDict* is a subclass of :class:`dict` for use when subclassing :class:`EnumType`. + + .. attribute:: EnumDict.member_names + + Return list of member names. + + .. versionadded:: 3.14 --------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 45cc1b5bad9b18..a291122aefc2ce 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -879,6 +879,11 @@ email (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the :cve:`2023-27043` fix.) +enum +---- + +* :class:`~enum.EnumDict` has been made public in :mod:`enum` to better support + subclassing :class:`~enum.EnumType`. fractions --------- From daa260ebb1c1b20321e7f26df7c9dbd35d4edcbf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Dec 2024 08:50:18 +0100 Subject: [PATCH 15/33] gh-128058: Fix test_builtin ImmortalTests (#128068) On 32-bit Free Threading systems, immortal reference count is 5 << 28, instead of 7 << 28. Co-authored-by: Peter Bierma --- Lib/test/test_builtin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index a92edad86839e6..f98138391bc1a8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2691,7 +2691,10 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = 7 << 28 + if support.Py_GIL_DISABLED: + IMMORTAL_REFCOUNT = 5 << 28 + else: + IMMORTAL_REFCOUNT = 7 << 28 else: IMMORTAL_REFCOUNT = 3 << 30 From 45e6dd63b88a782f2ec96ab1da54eb5a074d8f4c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 20 Dec 2024 00:22:26 -0800 Subject: [PATCH 16/33] gh-128030: Avoid error from PyModule_GetFilenameObject for non-module (#128047) I missed the extra `PyModule_Check` in #127660 because I was looking at 3.12 as the base implementation for import from. This meant that I missed the `PyModuleCheck` introduced in #112661. --- Lib/test/test_import/__init__.py | 23 +++++++++++++++++++ ...-12-17-22-28-15.gh-issue-128030.H1ptOD.rst | 1 + Python/ceval.c | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 83efbc1e25e77a..c2cec6444cb43a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -851,6 +851,29 @@ def test_frozen_module_from_import_error(self): stdout, stderr = popen.communicate() self.assertIn(expected_error, stdout) + def test_non_module_from_import_error(self): + prefix = """ +import sys +class NotAModule: ... +nm = NotAModule() +nm.symbol = 123 +sys.modules["not_a_module"] = nm +from not_a_module import symbol +""" + scripts = [ + prefix + "from not_a_module import missing_symbol", + prefix + "nm.__spec__ = []\nfrom not_a_module import missing_symbol", + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"ImportError: cannot import name 'missing_symbol' from " + b"'' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + def test_script_shadowing_stdlib(self): script_errors = [ ( diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst new file mode 100644 index 00000000000000..93d78632355b76 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst @@ -0,0 +1 @@ +Avoid error from calling ``PyModule_GetFilenameObject`` on a non-module object when importing a non-existent symbol from a non-module object. diff --git a/Python/ceval.c b/Python/ceval.c index fd891d7839151e..bfdf5687c287db 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2860,7 +2860,7 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) } } - if (origin == NULL) { + if (origin == NULL && PyModule_Check(v)) { // Fall back to __file__ for diagnostics if we don't have // an origin that is a location origin = PyModule_GetFilenameObject(v); From df46c780febab667ee01264ae32c4e866cecd911 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Dec 2024 11:57:44 +0000 Subject: [PATCH 17/33] GH-122548: Correct magic number comment (GH-128115) Correct magic number comment --- Include/internal/pycore_magic_number.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 079a9befcd4c5e..4c3b9c4c71da1b 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -262,7 +262,8 @@ Known values: Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) Python 3.14a1 3608 (Add support for slices) Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) - Python 3.14a3 3610 (Add NOT_TAKEN instruction) + (3610 accidentally omitted) + Python 3.14a4 3611 (Add NOT_TAKEN instruction) Python 3.15 will start with 3650 From 0974d7bb866062ed4aaa40f705d6cc4c294d99f1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Dec 2024 13:37:20 +0100 Subject: [PATCH 18/33] gh-109959: Log the current directory in test_glob.test_selflink() (#128122) --- Lib/test/test_glob.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index b72640bd871ba6..00187a3fb3537d 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -6,6 +6,7 @@ import unittest import warnings +from test import support from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -515,6 +516,12 @@ def test_selflink(self): os.makedirs(tempdir) self.addCleanup(shutil.rmtree, tempdir) with change_cwd(tempdir): + if support.verbose: + cwd = os.getcwd() + print(f"cwd: {cwd} ({len(cwd)} chars)") + cwdb = os.getcwdb() + print(f"cwdb: {cwdb!r} ({len(cwdb)} bytes)") + os.makedirs('dir') create_empty_file(os.path.join('dir', 'file')) os.symlink(os.curdir, os.path.join('dir', 'link')) From cbfe3023e46b544b80ea1a38a8c900c6fb881554 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Dec 2024 13:38:00 +0100 Subject: [PATCH 19/33] gh-128116: Skip test_socket VSOCK testStream() on PermissionError (#128120) --- Lib/test/test_socket.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 307d6e886c617f..aac213e36aecf0 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -547,7 +547,10 @@ def clientSetUp(self): self.cli.connect((cid, VSOCKPORT)) def testStream(self): - msg = self.conn.recv(1024) + try: + msg = self.conn.recv(1024) + except PermissionError as exc: + self.skipTest(repr(exc)) self.assertEqual(msg, MSG) def _testStream(self): From ba45e5cdd41a39ce0b3de08bdcfa9d8e28e0e4f3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 20 Dec 2024 08:02:46 -0500 Subject: [PATCH 20/33] gh-127946: Use a critical section for `CFuncPtr` attributes (GH-128109) --- Lib/test/test_ctypes/test_cfuncs.py | 20 +- ...-12-19-20-46-01.gh-issue-127946.4lM3Op.rst | 2 + Modules/_ctypes/_ctypes.c | 100 +++++++--- Modules/_ctypes/clinic/_ctypes.c.h | 174 +++++++++++++++++- 4 files changed, 266 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 48330c4b0a763b..e0c124607cb2e9 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -5,7 +5,8 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import import_helper +from test import support +from test.support import import_helper, threading_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -191,6 +192,23 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful on free-threading") + def test_thread_safety(self): + from threading import Thread + + def concurrent(): + for _ in range(100): + self._dll.tf_b.restype = c_byte + self._dll.tf_b.argtypes = (c_byte,) + + with threading_helper.catch_threading_exception() as exc: + with threading_helper.start_threads((Thread(target=concurrent) for _ in range(10))): + pass + + self.assertIsNone(exc.exc_value) + # The following repeats the above tests with stdcall functions (where # they are available) diff --git a/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst b/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst new file mode 100644 index 00000000000000..faf1ec042bc2b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst @@ -0,0 +1,2 @@ +Fix crash when modifying :class:`ctypes._CFuncPtr` objects concurrently on +the :term:`free threaded ` build. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 3a3b1da5084a67..dcdb7b2052a11e 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -128,8 +128,9 @@ bytes(cdata) /*[clinic input] module _ctypes +class _ctypes.CFuncPtr "PyCFuncPtrObject *" "&PyCFuncPtr_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=58e8c99474bc631e]*/ #define clinic_state() (get_module_state_by_class(cls)) #define clinic_state_sub() (get_module_state_by_class(cls->tp_base)) @@ -3422,21 +3423,37 @@ generic_pycdata_new(ctypes_state *st, PyCFuncPtr_Type */ +/*[clinic input] +@critical_section +@setter +_ctypes.CFuncPtr.errcheck +[clinic start generated code]*/ + static int -PyCFuncPtr_set_errcheck(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=6580cf1ffdf3b9fb input=84930bb16c490b33]*/ { - if (ob && !PyCallable_Check(ob)) { + if (value && !PyCallable_Check(value)) { PyErr_SetString(PyExc_TypeError, "the errcheck attribute must be callable"); return -1; } - Py_XINCREF(ob); - Py_XSETREF(self->errcheck, ob); + Py_XINCREF(value); + Py_XSETREF(self->errcheck, value); return 0; } +/*[clinic input] +@critical_section +@getter +_ctypes.CFuncPtr.errcheck + +a function to check for errors +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_errcheck(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_errcheck_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=dfa6fb5c6f90fd14 input=4672135fef37819f]*/ { if (self->errcheck) { return Py_NewRef(self->errcheck); @@ -3444,11 +3461,18 @@ PyCFuncPtr_get_errcheck(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) Py_RETURN_NONE; } +/*[clinic input] +@setter +@critical_section +_ctypes.CFuncPtr.restype +[clinic start generated code]*/ + static int -PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=0be0a086abbabf18 input=683c3bef4562ccc6]*/ { PyObject *checker, *oldchecker; - if (ob == NULL) { + if (value == NULL) { oldchecker = self->checker; self->checker = NULL; Py_CLEAR(self->restype); @@ -3457,27 +3481,36 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign } ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; - if (PyStgInfo_FromType(st, ob, &info) < 0) { + if (PyStgInfo_FromType(st, value, &info) < 0) { return -1; } - if (ob != Py_None && !info && !PyCallable_Check(ob)) { + if (value != Py_None && !info && !PyCallable_Check(value)) { PyErr_SetString(PyExc_TypeError, "restype must be a type, a callable, or None"); return -1; } - if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), &checker) < 0) { + if (PyObject_GetOptionalAttr(value, &_Py_ID(_check_retval_), &checker) < 0) { return -1; } oldchecker = self->checker; self->checker = checker; - Py_INCREF(ob); - Py_XSETREF(self->restype, ob); + Py_INCREF(value); + Py_XSETREF(self->restype, value); Py_XDECREF(oldchecker); return 0; } +/*[clinic input] +@getter +@critical_section +_ctypes.CFuncPtr.restype + +specify the result type +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_restype_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=c8f44cd16f1dee5e input=5e3ed95116204fd2]*/ { if (self->restype) { return Py_NewRef(self->restype); @@ -3495,28 +3528,44 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@setter +@critical_section +_ctypes.CFuncPtr.argtypes +[clinic start generated code]*/ + static int -PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=596a36e2ae89d7d1 input=c4627573e980aa8b]*/ { PyObject *converters; - if (ob == NULL || ob == Py_None) { + if (value == NULL || value == Py_None) { Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); - converters = converters_from_argtypes(st, ob); + converters = converters_from_argtypes(st, value); if (!converters) return -1; Py_XSETREF(self->converters, converters); - Py_INCREF(ob); - Py_XSETREF(self->argtypes, ob); + Py_INCREF(value); + Py_XSETREF(self->argtypes, value); } return 0; } +/*[clinic input] +@getter +@critical_section +_ctypes.CFuncPtr.argtypes + +specify the argument types +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_argtypes_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=c46b05a1b0f99172 input=37a8a545a56f8ae2]*/ { if (self->argtypes) { return Py_NewRef(self->argtypes); @@ -3535,13 +3584,9 @@ PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) } static PyGetSetDef PyCFuncPtr_getsets[] = { - { "errcheck", (getter)PyCFuncPtr_get_errcheck, (setter)PyCFuncPtr_set_errcheck, - "a function to check for errors", NULL }, - { "restype", (getter)PyCFuncPtr_get_restype, (setter)PyCFuncPtr_set_restype, - "specify the result type", NULL }, - { "argtypes", (getter)PyCFuncPtr_get_argtypes, - (setter)PyCFuncPtr_set_argtypes, - "specify the argument types", NULL }, + _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF + _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF + _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF { NULL, NULL } }; @@ -5054,7 +5099,6 @@ class _ctypes.Simple "PyObject *" "clinic_state()->Simple_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=016c476c7aa8b8a8]*/ - static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index 1332ba04cdfecd..405a3c9238d77d 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, @@ -601,6 +602,177 @@ PyCData_reduce(PyObject *myself, PyTypeObject *cls, PyObject *const *args, Py_ss return PyCData_reduce_impl(myself, cls); } +#if !defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# define _ctypes_CFuncPtr_errcheck_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, (setter)_ctypes_CFuncPtr_errcheck_set, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", NULL, (setter)_ctypes_CFuncPtr_errcheck_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_errcheck_set(PyCFuncPtrObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_errcheck_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_errcheck__doc__, +"a function to check for errors"); +#if defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# undef _ctypes_CFuncPtr_errcheck_DOCSTR +#endif +#define _ctypes_CFuncPtr_errcheck_DOCSTR _ctypes_CFuncPtr_errcheck__doc__ + +#if !defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# define _ctypes_CFuncPtr_errcheck_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, (setter)_ctypes_CFuncPtr_errcheck_set, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, NULL, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_errcheck_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_errcheck_get(PyCFuncPtrObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_errcheck_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CFuncPtr_restype_DOCSTR) +# define _ctypes_CFuncPtr_restype_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_RESTYPE_GETSETDEF) +# undef _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, (setter)_ctypes_CFuncPtr_restype_set, _ctypes_CFuncPtr_restype_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", NULL, (setter)_ctypes_CFuncPtr_restype_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_restype_set(PyCFuncPtrObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_restype_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_restype__doc__, +"specify the result type"); +#if defined(_ctypes_CFuncPtr_restype_DOCSTR) +# undef _ctypes_CFuncPtr_restype_DOCSTR +#endif +#define _ctypes_CFuncPtr_restype_DOCSTR _ctypes_CFuncPtr_restype__doc__ + +#if !defined(_ctypes_CFuncPtr_restype_DOCSTR) +# define _ctypes_CFuncPtr_restype_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_RESTYPE_GETSETDEF) +# undef _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, (setter)_ctypes_CFuncPtr_restype_set, _ctypes_CFuncPtr_restype_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, NULL, _ctypes_CFuncPtr_restype_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_restype_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_restype_get(PyCFuncPtrObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_restype_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# define _ctypes_CFuncPtr_argtypes_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, (setter)_ctypes_CFuncPtr_argtypes_set, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", NULL, (setter)_ctypes_CFuncPtr_argtypes_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_argtypes_set(PyCFuncPtrObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_argtypes_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_argtypes__doc__, +"specify the argument types"); +#if defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# undef _ctypes_CFuncPtr_argtypes_DOCSTR +#endif +#define _ctypes_CFuncPtr_argtypes_DOCSTR _ctypes_CFuncPtr_argtypes__doc__ + +#if !defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# define _ctypes_CFuncPtr_argtypes_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, (setter)_ctypes_CFuncPtr_argtypes_set, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, NULL, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_argtypes_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_argtypes_get(PyCFuncPtrObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_argtypes_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(Simple_from_outparm__doc__, "__ctypes_from_outparam__($self, /)\n" "--\n" @@ -621,4 +793,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py } return Simple_from_outparm_impl(self, cls); } -/*[clinic end generated code: output=52724c091e3a8b8d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cb3583522a2c5ce5 input=a9049054013a1b77]*/ From 78ffba4221dcb2e39fd5db80c297d1777588bb59 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 20 Dec 2024 14:28:18 +0100 Subject: [PATCH 21/33] gh-127295: ctypes: Switch field accessors to fixed-width integers (GH-127297) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should be a pure refactoring, without user-visible behaviour changes. Before this change, ctypes uses traditional native C types, usually identified by [`struct` format characters][struct-chars] when a short (and identifier-friendly) name is needed: - `signed char` (`b`) / `unsigned char` (`B`) - `short` (`h`) / `unsigned short` (`h`) - `int` (`i`) / `unsigned int` (`i`) - `long` (`l`) / `unsigned long` (`l`) - `long long` (`q`) / `unsigned long long` (`q`) These map to C99 fixed-width types, which this PR switches to: - - `int8_t`/`uint8_t` - `int16_t`/`uint16_t` - `int32_t`/`uint32_t` - `int64_t`/`uint64_t` The C standard doesn't guarantee that the “traditional” types must map to the fixints. But, [`ctypes` currently requires it][swapdefs], so the assumption won't break anything. By “map” I mean that the *size* of the types matches. The *alignment* requirements might not. This needs to be kept in mind but is not an issue in `ctypes` accessors, which [explicitly handle unaligned memory][memcpy] for the integer types. Note that there are 5 “traditional” C type sizes, but 4 fixed-width ones. Two of the former are functionally identical to one another; which ones they are is platform-specific (e.g. `int`==`long`==`int32_t`.) This means that one of the [current][current-impls-1] [implementations][current-impls-2] is redundant on any given platform. The fixint types are parametrized by the number of bytes/bits, and one bit for signedness. This makes it easier to autogenerate code for them or to write generic macros (though generic API like [`PyLong_AsNativeBytes`][PyLong_AsNativeBytes] is problematic for performance reasons -- especially compared to a `memcpy` with compile-time-constant size). When one has a *different* integer type, determining the corresponding fixint means a `sizeof` and signedness check. This is easier and more robust than the current implementations (see [`wchar_t`][sizeof-wchar_t] or [`_Bool`][sizeof-bool]). [swapdefs]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L420-L444 [struct-chars]: https://docs.python.org/3/library/struct.html#format-characters [current-impls-1]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L470-L653 [current-impls-2]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L703-L944 [memcpy]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L613 [PyLong_AsNativeBytes]: https://docs.python.org/3/c-api/long.html#c.PyLong_AsNativeBytes [sizeof-wchar_t]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L1547-L1555 [sizeof-bool]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L1562-L1572 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 4 +- Modules/_ctypes/callbacks.c | 2 +- Modules/_ctypes/cfield.c | 1132 +++++++++++++++-------------------- Modules/_ctypes/ctypes.h | 15 +- 4 files changed, 501 insertions(+), 652 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index dcdb7b2052a11e..ac520ffaad6c90 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1979,7 +1979,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; parg->pffi_type = &ffi_type_pointer; parg->tag = 'P'; - parg->obj = fd->setfunc(&parg->value, value, 0); + parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); return NULL; @@ -2444,7 +2444,7 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, parg->tag = fmt[0]; parg->pffi_type = fd->pffi_type; - parg->obj = fd->setfunc(&parg->value, value, 0); + parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) return (PyObject *)parg; PyObject *exc = PyErr_GetRaisedException(); diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 7b9f6437c7d55f..89c0749a093765 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -264,7 +264,7 @@ static void _CallPythonObject(ctypes_state *st, be the result. EXCEPT when restype is py_object - Python itself knows how to manage the refcount of these objects. */ - PyObject *keep = setfunc(mem, result, 0); + PyObject *keep = setfunc(mem, result, restype->size); if (keep == NULL) { /* Could not convert callback result. */ diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2b9e8a1a10d6f5..dcac9da75360a4 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -10,6 +10,7 @@ #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include // bool #include #include "ctypes.h" @@ -320,61 +321,6 @@ PyType_Spec cfield_spec = { }; -/******************************************************************/ -/* - Accessor functions -*/ - -/* Derived from Modules/structmodule.c: - Helper routine to get a Python integer and raise the appropriate error - if it isn't one */ - -static int -get_long(PyObject *v, long *p) -{ - long x = PyLong_AsUnsignedLongMask(v); - if (x == -1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling unsigned long */ - -static int -get_ulong(PyObject *v, unsigned long *p) -{ - unsigned long x = PyLong_AsUnsignedLongMask(v); - if (x == (unsigned long)-1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling native long long. */ - -static int -get_longlong(PyObject *v, long long *p) -{ - long long x = PyLong_AsUnsignedLongLongMask(v); - if (x == -1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling native unsigned long long. */ - -static int -get_ulonglong(PyObject *v, unsigned long long *p) -{ - unsigned long long x = PyLong_AsUnsignedLongLongMask(v); - if (x == (unsigned long long)-1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - /***************************************************************** * Integer fields, with bitfield support */ @@ -404,34 +350,8 @@ Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { /* This macro RETURNS the first parameter with the bit field CHANGED. */ #define SET(type, x, v, size) \ (NUM_BITS(size) ? \ - ( ( (type)x & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)v & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \ - : (type)v) - -#if SIZEOF_SHORT == 2 -# define SWAP_SHORT _Py_bswap16 -#else -# error "unsupported short size" -#endif - -#if SIZEOF_INT == 4 -# define SWAP_INT _Py_bswap32 -#else -# error "unsupported int size" -#endif - -#if SIZEOF_LONG == 4 -# define SWAP_LONG _Py_bswap32 -#elif SIZEOF_LONG == 8 -# define SWAP_LONG _Py_bswap64 -#else -# error "unsupported long size" -#endif - -#if SIZEOF_LONG_LONG == 8 -# define SWAP_LONG_LONG _Py_bswap64 -#else -# error "unsupported long long size" -#endif + ( ( (type)(x) & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)(v) & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \ + : (type)(v)) /***************************************************************** * The setter methods return an object which must be kept alive, to keep the @@ -454,203 +374,145 @@ Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { #endif /***************************************************************** - * integer accessor methods, supporting bit fields + * accessor methods for fixed-width integers (e.g. int8_t, uint64_t), + * supporting bit fields. + * These are named e.g. `i8_set`/`i8_get` or `u64_set`/`u64_get`, + * and are all alike, so they're defined using a macro. */ -static PyObject * -b_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - if (get_long(value, &val) < 0) - return NULL; - *(signed char *)ptr = SET(signed char, *(signed char *)ptr, val, size); - _RET(value); -} - - -static PyObject * -b_get(void *ptr, Py_ssize_t size) -{ - signed char val = *(signed char *)ptr; - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -B_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - if (get_ulong(value, &val) < 0) - return NULL; - *(unsigned char *)ptr = SET(unsigned char, *(unsigned char*)ptr, val, size); - _RET(value); -} - - -static PyObject * -B_get(void *ptr, Py_ssize_t size) -{ - unsigned char val = *(unsigned char *)ptr; - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -h_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - short x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(short, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - - -static PyObject * -h_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - short field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_SHORT(field); - field = SET(short, field, val, size); - field = SWAP_SHORT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -h_get(void *ptr, Py_ssize_t size) -{ - short val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong((long)val); -} - -static PyObject * -h_get_sw(void *ptr, Py_ssize_t size) -{ - short val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_SHORT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -H_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned short x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned short, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -H_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned short field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_SHORT(field); - field = SET(unsigned short, field, val, size); - field = SWAP_SHORT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -H_get(void *ptr, Py_ssize_t size) -{ - unsigned short val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -H_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned short val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_SHORT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -i_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - int x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(int, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -i_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - int field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_INT(field); - field = SET(int, field, val, size); - field = SWAP_INT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} +#define FIXINT_GETSET(TAG, CTYPE, NBITS, PYAPI_FROMFUNC) \ + static PyObject * \ + TAG ## _set(void *ptr, PyObject *value, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + if (PyLong_Check(value) \ + && PyUnstable_Long_IsCompact((PyLongObject *)value)) \ + { \ + val = (CTYPE)PyUnstable_Long_CompactValue( \ + (PyLongObject *)value); \ + } \ + else { \ + Py_ssize_t res = PyLong_AsNativeBytes( \ + value, &val, (NBITS) / 8, \ + Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + if (res < 0) { \ + return NULL; \ + } \ + } \ + CTYPE prev; \ + memcpy(&prev, ptr, (NBITS) / 8); \ + val = SET(CTYPE, prev, val, size_arg); \ + memcpy(ptr, &val, (NBITS) / 8); \ + _RET(value); \ + } \ + \ + static PyObject * \ + TAG ## _get(void *ptr, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + memcpy(&val, ptr, sizeof(val)); \ + GET_BITFIELD(val, size_arg); \ + return PYAPI_FROMFUNC(val); \ + } \ + /////////////////////////////////////////////////////////////////////////// + +/* Another macro for byte-swapped variants (e.g. `i8_set_sw`/`i8_get_sw`) */ + +#define FIXINT_GETSET_SW(TAG, CTYPE, NBITS, PYAPI_FROMFUNC, PY_SWAPFUNC) \ + static PyObject * \ + TAG ## _set_sw(void *ptr, PyObject *value, Py_ssize_t size_arg) \ + { \ + CTYPE val; \ + PyObject *res = TAG ## _set(&val, value, (NBITS) / 8); \ + if (res == NULL) { \ + return NULL; \ + } \ + Py_DECREF(res); \ + CTYPE field; \ + memcpy(&field, ptr, sizeof(field)); \ + field = PY_SWAPFUNC(field); \ + field = SET(CTYPE, field, val, size_arg); \ + field = PY_SWAPFUNC(field); \ + memcpy(ptr, &field, sizeof(field)); \ + _RET(value); \ + } \ + \ + static PyObject * \ + TAG ## _get_sw(void *ptr, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + memcpy(&val, ptr, sizeof(val)); \ + val = PY_SWAPFUNC(val); \ + GET_BITFIELD(val, size_arg); \ + return PYAPI_FROMFUNC(val); \ + } \ + /////////////////////////////////////////////////////////////////////////// + +/* These macros are expanded for all supported combinations of byte sizes + * (1, 2, 4, 8), signed and unsigned, native and swapped byteorder. + * That's a lot, so generate the list with Argument Clinic (`make clinic`). + */ +/*[python input] +for nbits in 8, 16, 32, 64: + for sgn in 'i', 'u': + u = 'u' if sgn == 'u' else '' + U = u.upper() + apibits = max(nbits, 32) + parts = [ + f'{sgn}{nbits}', + f'{u}int{nbits}_t', + f'{nbits}', + f'PyLong_From{U}Int{apibits}', + ] + print(f'FIXINT_GETSET({", ".join(parts)})') + if nbits > 8: + parts.append(f'_Py_bswap{nbits}') + print(f'FIXINT_GETSET_SW({", ".join(parts)})') +[python start generated code]*/ +FIXINT_GETSET(i8, int8_t, 8, PyLong_FromInt32) +FIXINT_GETSET(u8, uint8_t, 8, PyLong_FromUInt32) +FIXINT_GETSET(i16, int16_t, 16, PyLong_FromInt32) +FIXINT_GETSET_SW(i16, int16_t, 16, PyLong_FromInt32, _Py_bswap16) +FIXINT_GETSET(u16, uint16_t, 16, PyLong_FromUInt32) +FIXINT_GETSET_SW(u16, uint16_t, 16, PyLong_FromUInt32, _Py_bswap16) +FIXINT_GETSET(i32, int32_t, 32, PyLong_FromInt32) +FIXINT_GETSET_SW(i32, int32_t, 32, PyLong_FromInt32, _Py_bswap32) +FIXINT_GETSET(u32, uint32_t, 32, PyLong_FromUInt32) +FIXINT_GETSET_SW(u32, uint32_t, 32, PyLong_FromUInt32, _Py_bswap32) +FIXINT_GETSET(i64, int64_t, 64, PyLong_FromInt64) +FIXINT_GETSET_SW(i64, int64_t, 64, PyLong_FromInt64, _Py_bswap64) +FIXINT_GETSET(u64, uint64_t, 64, PyLong_FromUInt64) +FIXINT_GETSET_SW(u64, uint64_t, 64, PyLong_FromUInt64, _Py_bswap64) +/*[python end generated code: output=3d60c96fa58e07d5 input=0b7e166f2ea18e70]*/ + +// For one-byte types, swapped variants are the same as native +#define i8_set_sw i8_set +#define i8_get_sw i8_get +#define u8_set_sw u8_set +#define u8_get_sw u8_get + +#undef FIXINT_GETSET +#undef FIXINT_GETSET_SW -static PyObject * -i_get(void *ptr, Py_ssize_t size) -{ - int val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -i_get_sw(void *ptr, Py_ssize_t size) -{ - int val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_INT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} +/***************************************************************** + * non-integer accessor methods, not supporting bit fields + */ #ifndef MS_WIN32 /* http://msdn.microsoft.com/en-us/library/cc237864.aspx */ #define VARIANT_FALSE 0x0000 #define VARIANT_TRUE 0xFFFF #endif -/* short BOOL - VARIANT_BOOL */ +/* v: short BOOL - VARIANT_BOOL */ static PyObject * -vBOOL_set(void *ptr, PyObject *value, Py_ssize_t size) +v_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(short int))); switch (PyObject_IsTrue(value)) { case -1: return NULL; @@ -664,22 +526,25 @@ vBOOL_set(void *ptr, PyObject *value, Py_ssize_t size) } static PyObject * -vBOOL_get(void *ptr, Py_ssize_t size) +v_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(short int))); return PyBool_FromLong((long)*(short int *)ptr); } +/* bool ('?'): bool (i.e. _Bool) */ static PyObject * bool_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(bool))); switch (PyObject_IsTrue(value)) { case -1: return NULL; case 0: - *(_Bool *)ptr = 0; + *(bool *)ptr = 0; _RET(value); default: - *(_Bool *)ptr = 1; + *(bool *)ptr = 1; _RET(value); } } @@ -687,260 +552,15 @@ bool_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * bool_get(void *ptr, Py_ssize_t size) { - return PyBool_FromLong((long)*(_Bool *)ptr); -} - -static PyObject * -I_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned int x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned int, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -I_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned int field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_INT(field); - field = SET(unsigned int, field, (unsigned int)val, size); - field = SWAP_INT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -I_get(void *ptr, Py_ssize_t size) -{ - unsigned int val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -I_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned int val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_INT(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -l_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - long x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -l_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - long field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG(field); - field = SET(long, field, val, size); - field = SWAP_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -l_get(void *ptr, Py_ssize_t size) -{ - long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -l_get_sw(void *ptr, Py_ssize_t size) -{ - long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -L_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned long x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -L_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned long field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG(field); - field = SET(unsigned long, field, val, size); - field = SWAP_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -L_get(void *ptr, Py_ssize_t size) -{ - unsigned long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -L_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -q_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long long val; - long long x; - if (get_longlong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -q_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long long val; - long long field; - if (get_longlong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG_LONG(field); - field = SET(long long, field, val, size); - field = SWAP_LONG_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -q_get(void *ptr, Py_ssize_t size) -{ - long long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLongLong(val); -} - -static PyObject * -q_get_sw(void *ptr, Py_ssize_t size) -{ - long long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromLongLong(val); -} - -static PyObject * -Q_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long long val; - unsigned long long x; - if (get_ulonglong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -Q_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long long val; - unsigned long long field; - if (get_ulonglong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG_LONG(field); - field = SET(unsigned long long, field, val, size); - field = SWAP_LONG_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -Q_get(void *ptr, Py_ssize_t size) -{ - unsigned long long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLongLong(val); -} - -static PyObject * -Q_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned long long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLongLong(val); + assert(NUM_BITS(size) || (size == sizeof(bool))); + return PyBool_FromLong((long)*(bool *)ptr); } -/***************************************************************** - * non-integer accessor methods, not supporting bit fields - */ - - +/* g: long double */ static PyObject * g_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double))); long double x; x = PyFloat_AsDouble(value); @@ -953,14 +573,17 @@ g_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * g_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double))); long double val; memcpy(&val, ptr, sizeof(long double)); return PyFloat_FromDouble(val); } +/* d: double */ static PyObject * d_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double x; x = PyFloat_AsDouble(value); @@ -973,15 +596,18 @@ d_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * d_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double val; memcpy(&val, ptr, sizeof(val)); return PyFloat_FromDouble(val); } #if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +/* C: double complex */ static PyObject * C_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double complex))); Py_complex c = PyComplex_AsCComplex(value); if (c.real == -1 && PyErr_Occurred()) { @@ -995,15 +621,18 @@ C_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * C_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double complex))); double complex x; memcpy(&x, ptr, sizeof(x)); return PyComplex_FromDoubles(creal(x), cimag(x)); } +/* E: float complex */ static PyObject * E_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float complex))); Py_complex c = PyComplex_AsCComplex(value); if (c.real == -1 && PyErr_Occurred()) { @@ -1017,15 +646,18 @@ E_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * E_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float complex))); float complex x; memcpy(&x, ptr, sizeof(x)); return PyComplex_FromDoubles(crealf(x), cimagf(x)); } +/* F: long double complex */ static PyObject * F_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double complex))); Py_complex c = PyComplex_AsCComplex(value); if (c.real == -1 && PyErr_Occurred()) { @@ -1039,6 +671,7 @@ F_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * F_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double complex))); long double complex x; memcpy(&x, ptr, sizeof(x)); @@ -1046,9 +679,11 @@ F_get(void *ptr, Py_ssize_t size) } #endif +/* d: double */ static PyObject * d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double x; x = PyFloat_AsDouble(value); @@ -1067,6 +702,7 @@ d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * d_get_sw(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); #ifdef WORDS_BIGENDIAN return PyFloat_FromDouble(PyFloat_Unpack8(ptr, 1)); #else @@ -1074,9 +710,11 @@ d_get_sw(void *ptr, Py_ssize_t size) #endif } +/* f: float */ static PyObject * f_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float x; x = (float)PyFloat_AsDouble(value); @@ -1089,6 +727,7 @@ f_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * f_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float val; memcpy(&val, ptr, sizeof(val)); return PyFloat_FromDouble(val); @@ -1097,6 +736,7 @@ f_get(void *ptr, Py_ssize_t size) static PyObject * f_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float x; x = (float)PyFloat_AsDouble(value); @@ -1115,6 +755,7 @@ f_set_sw(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * f_get_sw(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); #ifdef WORDS_BIGENDIAN return PyFloat_FromDouble(PyFloat_Unpack4(ptr, 1)); #else @@ -1122,6 +763,7 @@ f_get_sw(void *ptr, Py_ssize_t size) #endif } +/* O: Python object */ /* py_object refcounts: @@ -1135,6 +777,7 @@ f_get_sw(void *ptr, Py_ssize_t size) static PyObject * O_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(PyObject *))); PyObject *ob = *(PyObject **)ptr; if (ob == NULL) { if (!PyErr_Occurred()) @@ -1149,15 +792,18 @@ O_get(void *ptr, Py_ssize_t size) static PyObject * O_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(PyObject *))); /* Hm, does the memory block need it's own refcount or not? */ *(PyObject **)ptr = value; return Py_NewRef(value); } +/* c: a single byte-character */ static PyObject * c_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(char))); if (PyBytes_Check(value)) { if (PyBytes_GET_SIZE(value) != 1) { PyErr_Format(PyExc_TypeError, @@ -1204,13 +850,15 @@ c_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * c_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(char))); return PyBytes_FromStringAndSize((char *)ptr, 1); } -/* u - a single wchar_t character */ +/* u: a single wchar_t character */ static PyObject * u_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(wchar_t))); Py_ssize_t len; wchar_t chars[2]; if (!PyUnicode_Check(value)) { @@ -1244,10 +892,11 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * u_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(wchar_t))); return PyUnicode_FromWideChar((wchar_t *)ptr, 1); } -/* U - a unicode string */ +/* U: a wchar_t* unicode string */ static PyObject * U_get(void *ptr, Py_ssize_t size) { @@ -1306,6 +955,7 @@ U_set(void *ptr, PyObject *value, Py_ssize_t length) } +/* s: a byte string */ static PyObject * s_get(void *ptr, Py_ssize_t size) { @@ -1355,6 +1005,7 @@ s_set(void *ptr, PyObject *value, Py_ssize_t length) _RET(value); } +/* z: a byte string, can be set from integer pointer */ static PyObject * z_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1391,6 +1042,7 @@ z_get(void *ptr, Py_ssize_t size) } } +/* Z: a wchar* string, can be set from integer pointer */ static PyObject * Z_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1445,8 +1097,9 @@ Z_get(void *ptr, Py_ssize_t size) #ifdef MS_WIN32 +/* X: COM BSTR (wide-char string to be handled handled using Windows API) */ static PyObject * -BSTR_set(void *ptr, PyObject *value, Py_ssize_t size) +X_set(void *ptr, PyObject *value, Py_ssize_t size) { BSTR bstr; @@ -1490,7 +1143,7 @@ BSTR_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * -BSTR_get(void *ptr, Py_ssize_t size) +X_get(void *ptr, Py_ssize_t size) { BSTR p; p = *(BSTR *)ptr; @@ -1505,9 +1158,11 @@ BSTR_get(void *ptr, Py_ssize_t size) } #endif +/* P: generic pointer */ static PyObject * P_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(void *))); void *v; if (value == Py_None) { *(void **)ptr = NULL; @@ -1539,154 +1194,339 @@ P_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * P_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(void *))); if (*(void **)ptr == NULL) { Py_RETURN_NONE; } return PyLong_FromVoidPtr(*(void **)ptr); } -static struct fielddesc formattable[] = { - { 's', s_set, s_get, NULL}, - { 'b', b_set, b_get, NULL}, - { 'B', B_set, B_get, NULL}, - { 'c', c_set, c_get, NULL}, - { 'd', d_set, d_get, NULL, d_set_sw, d_get_sw}, -#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) - { 'C', C_set, C_get, NULL}, - { 'E', E_set, E_get, NULL}, - { 'F', F_set, F_get, NULL}, -#endif - { 'g', g_set, g_get, NULL}, - { 'f', f_set, f_get, NULL, f_set_sw, f_get_sw}, - { 'h', h_set, h_get, NULL, h_set_sw, h_get_sw}, - { 'H', H_set, H_get, NULL, H_set_sw, H_get_sw}, - { 'i', i_set, i_get, NULL, i_set_sw, i_get_sw}, - { 'I', I_set, I_get, NULL, I_set_sw, I_get_sw}, - { 'l', l_set, l_get, NULL, l_set_sw, l_get_sw}, - { 'L', L_set, L_get, NULL, L_set_sw, L_get_sw}, - { 'q', q_set, q_get, NULL, q_set_sw, q_get_sw}, - { 'Q', Q_set, Q_get, NULL, Q_set_sw, Q_get_sw}, - { 'P', P_set, P_get, NULL}, - { 'z', z_set, z_get, NULL}, - { 'u', u_set, u_get, NULL}, - { 'U', U_set, U_get, NULL}, - { 'Z', Z_set, Z_get, NULL}, -#ifdef MS_WIN32 - { 'X', BSTR_set, BSTR_get, NULL}, -#endif - { 'v', vBOOL_set, vBOOL_get, NULL}, -#if SIZEOF__BOOL == SIZEOF_INT - { '?', bool_set, bool_get, NULL, I_set_sw, I_get_sw}, -#elif SIZEOF__BOOL == SIZEOF_LONG - { '?', bool_set, bool_get, NULL, L_set_sw, L_get_sw}, -#elif SIZEOF__BOOL == SIZEOF_LONG_LONG - { '?', bool_set, bool_get, NULL, Q_set_sw, Q_get_sw}, -#else - { '?', bool_set, bool_get, NULL}, -#endif /* SIZEOF__BOOL */ - { 'O', O_set, O_get, NULL}, - { 0, NULL, NULL, NULL}, +/* Table with info about all formats. + * Must be accessed via _ctypes_get_fielddesc, which initializes it on + * first use. After initialization it's treated as constant & read-only. + */ + +struct formattable { +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + print(f' struct fielddesc fmt_{sgn}{nbytes};') +for code in 'sbBcdCEFgfhHiIlLqQPzuUZXvO': + print(f' struct fielddesc fmt_{code};') +[python start generated code]*/ + struct fielddesc fmt_i8; + struct fielddesc fmt_u8; + struct fielddesc fmt_i16; + struct fielddesc fmt_u16; + struct fielddesc fmt_i32; + struct fielddesc fmt_u32; + struct fielddesc fmt_i64; + struct fielddesc fmt_u64; + struct fielddesc fmt_s; + struct fielddesc fmt_b; + struct fielddesc fmt_B; + struct fielddesc fmt_c; + struct fielddesc fmt_d; + struct fielddesc fmt_C; + struct fielddesc fmt_E; + struct fielddesc fmt_F; + struct fielddesc fmt_g; + struct fielddesc fmt_f; + struct fielddesc fmt_h; + struct fielddesc fmt_H; + struct fielddesc fmt_i; + struct fielddesc fmt_I; + struct fielddesc fmt_l; + struct fielddesc fmt_L; + struct fielddesc fmt_q; + struct fielddesc fmt_Q; + struct fielddesc fmt_P; + struct fielddesc fmt_z; + struct fielddesc fmt_u; + struct fielddesc fmt_U; + struct fielddesc fmt_Z; + struct fielddesc fmt_X; + struct fielddesc fmt_v; + struct fielddesc fmt_O; +/*[python end generated code: output=fa648744ec7f919d input=087d58357d4bf2c5]*/ + + // bool has code '?': + struct fielddesc fmt_bool; + + // always contains NULLs: + struct fielddesc fmt_nil; }; -/* - Ideas: Implement VARIANT in this table, using 'V' code. - Use '?' as code for BOOL. -*/ +static struct formattable formattable; + + +/* Get fielddesc info for a fixed-width integer. + * N.B: - must be called after (or from) _ctypes_init_fielddesc! + * - nbytes must be one of the supported values + */ + +static inline struct fielddesc * +_ctypes_fixint_fielddesc(Py_ssize_t nbytes, bool is_signed) +{ +#define _PACK(NBYTES, SGN) ((NBYTES<<2) + (SGN ? 1 : 0)) + switch (_PACK(nbytes, is_signed)) { +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + is_signed = sgn == 'i' + print(f' case (_PACK({nbytes // 8}, {int(is_signed)})): ' + + f'return &formattable.fmt_{sgn}{nbytes};') +[python start generated code]*/ + case (_PACK(1, 1)): return &formattable.fmt_i8; + case (_PACK(1, 0)): return &formattable.fmt_u8; + case (_PACK(2, 1)): return &formattable.fmt_i16; + case (_PACK(2, 0)): return &formattable.fmt_u16; + case (_PACK(4, 1)): return &formattable.fmt_i32; + case (_PACK(4, 0)): return &formattable.fmt_u32; + case (_PACK(8, 1)): return &formattable.fmt_i64; + case (_PACK(8, 0)): return &formattable.fmt_u64; +/*[python end generated code: output=0194ba35c4d64ff3 input=ee9f6f5bb872d645]*/ +#undef _PACK + } + /* ctypes currently only supports platforms where the basic integer types + * (`char`, `short`, `int`, `long`, `long long`) have 1, 2, 4, or 8 bytes + * (i.e. 8 to 64 bits). + */ + Py_UNREACHABLE(); +} + + +/* Macro to call _ctypes_fixint_fielddesc for a given C type. */ + +_Py_COMP_DIAG_PUSH +#if defined(__GNUC__) && (__GNUC__ < 14) +/* The signedness check expands to an expression that's always true or false. + * Older GCC gives a '-Wtype-limits' warning for this, which is a GCC bug + * (docs say it should "not warn for constant expressions"): + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86647 + * Silence that warning. + */ +#pragma GCC diagnostic ignored "-Wtype-limits" +#endif + +#define FIXINT_FIELDDESC_FOR(C_TYPE) \ + _ctypes_fixint_fielddesc(sizeof(C_TYPE), (C_TYPE)-1 < 0) + /* Delayed initialization. Windows cannot statically reference dynamically loaded addresses from DLLs. */ void _ctypes_init_fielddesc(void) { - struct fielddesc *fd = formattable; - for (; fd->code; ++fd) { - switch (fd->code) { - case 's': fd->pffi_type = &ffi_type_pointer; break; - case 'b': fd->pffi_type = &ffi_type_schar; break; - case 'B': fd->pffi_type = &ffi_type_uchar; break; - case 'c': fd->pffi_type = &ffi_type_schar; break; - case 'd': fd->pffi_type = &ffi_type_double; break; + /* Fixed-width integers */ + +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + is_signed = sgn == 'i' + u = 'u' if sgn == 'u' else 's' + parts = [ + f"0", + f'&ffi_type_{u}int{nbytes}', + f'{sgn}{nbytes}_set', + f'{sgn}{nbytes}_get', + f'{sgn}{nbytes}_set_sw', + f'{sgn}{nbytes}_get_sw', + ] + print(f' formattable.fmt_{sgn}{nbytes} = (struct fielddesc){{') + print(f' {', '.join(parts)} }};') +[python start generated code]*/ + formattable.fmt_i8 = (struct fielddesc){ + 0, &ffi_type_sint8, i8_set, i8_get, i8_set_sw, i8_get_sw }; + formattable.fmt_u8 = (struct fielddesc){ + 0, &ffi_type_uint8, u8_set, u8_get, u8_set_sw, u8_get_sw }; + formattable.fmt_i16 = (struct fielddesc){ + 0, &ffi_type_sint16, i16_set, i16_get, i16_set_sw, i16_get_sw }; + formattable.fmt_u16 = (struct fielddesc){ + 0, &ffi_type_uint16, u16_set, u16_get, u16_set_sw, u16_get_sw }; + formattable.fmt_i32 = (struct fielddesc){ + 0, &ffi_type_sint32, i32_set, i32_get, i32_set_sw, i32_get_sw }; + formattable.fmt_u32 = (struct fielddesc){ + 0, &ffi_type_uint32, u32_set, u32_get, u32_set_sw, u32_get_sw }; + formattable.fmt_i64 = (struct fielddesc){ + 0, &ffi_type_sint64, i64_set, i64_get, i64_set_sw, i64_get_sw }; + formattable.fmt_u64 = (struct fielddesc){ + 0, &ffi_type_uint64, u64_set, u64_get, u64_set_sw, u64_get_sw }; +/*[python end generated code: output=16806fe0ca3a9c4c input=850b8dd6388b1b10]*/ + + + /* Native C integers. + * These use getters/setters for fixed-width ints but have their own + * `code` and `pffi_type`. + */ + +/*[python input] +for base_code, base_c_type in [ + ('b', 'char'), + ('h', 'short'), + ('i', 'int'), + ('l', 'long'), + ('q', 'long long'), +]: + for code, c_type, ffi_type in [ + (base_code, 'signed ' + base_c_type, 's' + base_c_type), + (base_code.upper(), 'unsigned ' + base_c_type, 'u' + base_c_type), + ]: + print(f' formattable.fmt_{code} = *FIXINT_FIELDDESC_FOR({c_type});') + print(f" formattable.fmt_{code}.code = '{code}';") + if base_code == 'q': + # ffi doesn't have `long long`; keep use the fixint type + pass + else: + print(f' formattable.fmt_{code}.pffi_type = &ffi_type_{ffi_type};') +[python start generated code]*/ + formattable.fmt_b = *FIXINT_FIELDDESC_FOR(signed char); + formattable.fmt_b.code = 'b'; + formattable.fmt_b.pffi_type = &ffi_type_schar; + formattable.fmt_B = *FIXINT_FIELDDESC_FOR(unsigned char); + formattable.fmt_B.code = 'B'; + formattable.fmt_B.pffi_type = &ffi_type_uchar; + formattable.fmt_h = *FIXINT_FIELDDESC_FOR(signed short); + formattable.fmt_h.code = 'h'; + formattable.fmt_h.pffi_type = &ffi_type_sshort; + formattable.fmt_H = *FIXINT_FIELDDESC_FOR(unsigned short); + formattable.fmt_H.code = 'H'; + formattable.fmt_H.pffi_type = &ffi_type_ushort; + formattable.fmt_i = *FIXINT_FIELDDESC_FOR(signed int); + formattable.fmt_i.code = 'i'; + formattable.fmt_i.pffi_type = &ffi_type_sint; + formattable.fmt_I = *FIXINT_FIELDDESC_FOR(unsigned int); + formattable.fmt_I.code = 'I'; + formattable.fmt_I.pffi_type = &ffi_type_uint; + formattable.fmt_l = *FIXINT_FIELDDESC_FOR(signed long); + formattable.fmt_l.code = 'l'; + formattable.fmt_l.pffi_type = &ffi_type_slong; + formattable.fmt_L = *FIXINT_FIELDDESC_FOR(unsigned long); + formattable.fmt_L.code = 'L'; + formattable.fmt_L.pffi_type = &ffi_type_ulong; + formattable.fmt_q = *FIXINT_FIELDDESC_FOR(signed long long); + formattable.fmt_q.code = 'q'; + formattable.fmt_Q = *FIXINT_FIELDDESC_FOR(unsigned long long); + formattable.fmt_Q.code = 'Q'; +/*[python end generated code: output=873c87a2e6b5075a input=ee814ca263aac18e]*/ + + + /* Other types have bespoke setters and getters named `@_set` and `@_get`, + * where `@` is the type code. + * Some have swapped variants, `@_set_sw` and `@_get_sw` + */ + +#define _TABLE_ENTRY(SYMBOL, FFI_TYPE, ...) \ + formattable.fmt_ ## SYMBOL = \ + (struct fielddesc){(#SYMBOL)[0], (FFI_TYPE), __VA_ARGS__}; \ + /////////////////////////////////////////////////////////////////////////// + +#define TABLE_ENTRY(SYMBOL, FFI_TYPE) \ + _TABLE_ENTRY(SYMBOL, FFI_TYPE, SYMBOL ## _set, SYMBOL ## _get) \ + /////////////////////////////////////////////////////////////////////////// + +#define TABLE_ENTRY_SW(SYMBOL, FFI_TYPE) \ + _TABLE_ENTRY(SYMBOL, FFI_TYPE, SYMBOL ## _set, \ + SYMBOL ## _get, SYMBOL ## _set_sw, SYMBOL ## _get_sw) \ + /////////////////////////////////////////////////////////////////////////// + + TABLE_ENTRY_SW(d, &ffi_type_double); #if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) - case 'C': fd->pffi_type = &ffi_type_complex_double; break; - case 'E': fd->pffi_type = &ffi_type_complex_float; break; - case 'F': fd->pffi_type = &ffi_type_complex_longdouble; break; + TABLE_ENTRY(C, &ffi_type_complex_double); + TABLE_ENTRY(E, &ffi_type_complex_float); + TABLE_ENTRY(F, &ffi_type_complex_longdouble); #endif - case 'g': fd->pffi_type = &ffi_type_longdouble; break; - case 'f': fd->pffi_type = &ffi_type_float; break; - case 'h': fd->pffi_type = &ffi_type_sshort; break; - case 'H': fd->pffi_type = &ffi_type_ushort; break; - case 'i': fd->pffi_type = &ffi_type_sint; break; - case 'I': fd->pffi_type = &ffi_type_uint; break; - /* XXX Hm, sizeof(int) == sizeof(long) doesn't hold on every platform */ - /* As soon as we can get rid of the type codes, this is no longer a problem */ - #if SIZEOF_LONG == 4 - case 'l': fd->pffi_type = &ffi_type_sint32; break; - case 'L': fd->pffi_type = &ffi_type_uint32; break; - #elif SIZEOF_LONG == 8 - case 'l': fd->pffi_type = &ffi_type_sint64; break; - case 'L': fd->pffi_type = &ffi_type_uint64; break; - #else - #error - #endif - #if SIZEOF_LONG_LONG == 8 - case 'q': fd->pffi_type = &ffi_type_sint64; break; - case 'Q': fd->pffi_type = &ffi_type_uint64; break; - #else - #error - #endif - case 'P': fd->pffi_type = &ffi_type_pointer; break; - case 'z': fd->pffi_type = &ffi_type_pointer; break; - case 'u': - if (sizeof(wchar_t) == sizeof(short)) - fd->pffi_type = &ffi_type_sshort; - else if (sizeof(wchar_t) == sizeof(int)) - fd->pffi_type = &ffi_type_sint; - else if (sizeof(wchar_t) == sizeof(long)) - fd->pffi_type = &ffi_type_slong; - else - Py_UNREACHABLE(); - break; - case 'U': fd->pffi_type = &ffi_type_pointer; break; - case 'Z': fd->pffi_type = &ffi_type_pointer; break; - #ifdef MS_WIN32 - case 'X': fd->pffi_type = &ffi_type_pointer; break; - #endif - case 'v': fd->pffi_type = &ffi_type_sshort; break; - #if SIZEOF__BOOL == 1 - case '?': fd->pffi_type = &ffi_type_uchar; break; /* Also fallback for no native _Bool support */ - #elif SIZEOF__BOOL == SIZEOF_SHORT - case '?': fd->pffi_type = &ffi_type_ushort; break; - #elif SIZEOF__BOOL == SIZEOF_INT - case '?': fd->pffi_type = &ffi_type_uint; break; - #elif SIZEOF__BOOL == SIZEOF_LONG - case '?': fd->pffi_type = &ffi_type_ulong; break; - #elif SIZEOF__BOOL == SIZEOF_LONG_LONG - case '?': fd->pffi_type = &ffi_type_ulong; break; - #endif /* SIZEOF__BOOL */ - case 'O': fd->pffi_type = &ffi_type_pointer; break; - default: - Py_UNREACHABLE(); - } - } + TABLE_ENTRY(g, &ffi_type_longdouble); + TABLE_ENTRY_SW(f, &ffi_type_float); + TABLE_ENTRY(v, &ffi_type_sshort); /* vBOOL */ + + // ctypes.c_char is signed for FFI, even where C wchar_t is unsigned. + TABLE_ENTRY(c, _ctypes_fixint_fielddesc(sizeof(char), true)->pffi_type); + // ctypes.c_wchar is signed for FFI, even where C wchar_t is unsigned. + TABLE_ENTRY(u, _ctypes_fixint_fielddesc(sizeof(wchar_t), true)->pffi_type); + + TABLE_ENTRY(s, &ffi_type_pointer); + TABLE_ENTRY(P, &ffi_type_pointer); + TABLE_ENTRY(z, &ffi_type_pointer); + TABLE_ENTRY(U, &ffi_type_pointer); + TABLE_ENTRY(Z, &ffi_type_pointer); +#ifdef MS_WIN32 + TABLE_ENTRY(X, &ffi_type_pointer); +#endif + TABLE_ENTRY(O, &ffi_type_pointer); + +#undef TABLE_ENTRY_SW +#undef TABLE_ENTRY +#undef _TABLE_ENTRY + /* bool has code '?', fill it in manually */ + + // ctypes.c_bool is unsigned for FFI, even where C bool is signed. + formattable.fmt_bool = *_ctypes_fixint_fielddesc(sizeof(bool), false); + formattable.fmt_bool.code = '?'; + formattable.fmt_bool.setfunc = bool_set; + formattable.fmt_bool.getfunc = bool_get; } +#undef FIXINT_FIELDDESC_FOR +_Py_COMP_DIAG_POP struct fielddesc * _ctypes_get_fielddesc(const char *fmt) { - static int initialized = 0; - struct fielddesc *table = formattable; - + static bool initialized = false; + static PyMutex mutex = {0}; + PyMutex_Lock(&mutex); if (!initialized) { - initialized = 1; _ctypes_init_fielddesc(); + initialized = true; } - - for (; table->code; ++table) { - if (table->code == fmt[0]) - return table; + PyMutex_Unlock(&mutex); + struct fielddesc *result = NULL; + switch(fmt[0]) { +/*[python input] +for code in 'sbBcdCEFgfhHiIlLqQPzuUZXvO': + print(f" case '{code}': result = &formattable.fmt_{code}; break;") +[python start generated code]*/ + case 's': result = &formattable.fmt_s; break; + case 'b': result = &formattable.fmt_b; break; + case 'B': result = &formattable.fmt_B; break; + case 'c': result = &formattable.fmt_c; break; + case 'd': result = &formattable.fmt_d; break; + case 'C': result = &formattable.fmt_C; break; + case 'E': result = &formattable.fmt_E; break; + case 'F': result = &formattable.fmt_F; break; + case 'g': result = &formattable.fmt_g; break; + case 'f': result = &formattable.fmt_f; break; + case 'h': result = &formattable.fmt_h; break; + case 'H': result = &formattable.fmt_H; break; + case 'i': result = &formattable.fmt_i; break; + case 'I': result = &formattable.fmt_I; break; + case 'l': result = &formattable.fmt_l; break; + case 'L': result = &formattable.fmt_L; break; + case 'q': result = &formattable.fmt_q; break; + case 'Q': result = &formattable.fmt_Q; break; + case 'P': result = &formattable.fmt_P; break; + case 'z': result = &formattable.fmt_z; break; + case 'u': result = &formattable.fmt_u; break; + case 'U': result = &formattable.fmt_U; break; + case 'Z': result = &formattable.fmt_Z; break; + case 'X': result = &formattable.fmt_X; break; + case 'v': result = &formattable.fmt_v; break; + case 'O': result = &formattable.fmt_O; break; +/*[python end generated code: output=81a8223dda9f81f7 input=2f59666d3c024edf]*/ + case '?': result = &formattable.fmt_bool; break; } - return NULL; + if (!result || !result->code) { + return NULL; + } + assert(result->pffi_type); + assert(result->setfunc); + assert(result->getfunc); + return result; } +/* + Ideas: Implement VARIANT in this table, using 'V' code. +*/ + /*---------------- EOF ----------------*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7e0804054cded4..45e00a538fb5a5 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -113,8 +113,17 @@ extern PyType_Spec cthunk_spec; typedef struct tagPyCArgObject PyCArgObject; typedef struct tagCDataObject CDataObject; -typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); -typedef PyObject *(* SETFUNC)(void *, PyObject *value, Py_ssize_t size); + +// GETFUNC: convert the C value at *ptr* to Python object, return the object +// SETFUNC: write content of the PyObject *value* to the location at *ptr*; +// return a new reference to either *value*, or None for simple types +// (see _CTYPES_DEBUG_KEEP). +// Note that the *size* arg can have different meanings depending on context: +// for string-like arrays it's the size in bytes +// for int-style fields it's either the type size, or bitfiled info +// that can be unpacked using the LOW_BIT & NUM_BITS macros. +typedef PyObject *(* GETFUNC)(void *ptr, Py_ssize_t size); +typedef PyObject *(* SETFUNC)(void *ptr, PyObject *value, Py_ssize_t size); typedef PyCArgObject *(* PARAMFUNC)(ctypes_state *st, CDataObject *obj); /* A default buffer in CDataObject, which can be used for small C types. If @@ -239,9 +248,9 @@ extern CThunkObject *_ctypes_alloc_callback(ctypes_state *st, /* a table entry describing a predefined ctypes type */ struct fielddesc { char code; + ffi_type *pffi_type; /* always statically allocated */ SETFUNC setfunc; GETFUNC getfunc; - ffi_type *pffi_type; /* always statically allocated */ SETFUNC setfunc_swapped; GETFUNC getfunc_swapped; }; From 128cc47fbd44e3e09c50d9674fe4a4bba3be450c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Dec 2024 16:52:20 +0000 Subject: [PATCH 22/33] GH-127705: Add debug mode for `_PyStackRef`s inspired by HPy debug mode (GH-128121) --- Include/internal/pycore_interp.h | 6 + Include/internal/pycore_stackref.h | 113 +++++++++++++ Makefile.pre.in | 1 + ...-12-20-12-25-16.gh-issue-127705.WmCz1z.rst | 4 + Objects/frameobject.c | 4 +- Python/bytecodes.c | 17 +- Python/ceval.c | 67 ++++++-- Python/ceval_macros.h | 6 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 23 ++- Python/pystate.c | 29 ++++ Python/stackrefs.c | 156 ++++++++++++++++++ 12 files changed, 395 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst create mode 100644 Python/stackrefs.c diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 87cdcb5b119d15..a3c14dceffd7a0 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -34,6 +34,7 @@ extern "C" { #include "pycore_optimizer.h" // _PyOptimizerObject #include "pycore_obmalloc.h" // struct _obmalloc_state #include "pycore_qsbr.h" // struct _qsbr_state +#include "pycore_stackref.h" // Py_STACKREF_DEBUG #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_uniqueid.h" // struct _Py_unique_id_pool @@ -285,6 +286,11 @@ struct _is { _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. // See https://github.com/python/cpython/issues/127117. + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + uint64_t next_stackref; + _Py_hashtable_t *stackref_debug_table; +#endif }; diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 90a3118352f7ae..1ae62cc69bb364 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -4,6 +4,9 @@ extern "C" { #endif +// Define this to get precise tracking of stackrefs. +// #define Py_STACKREF_DEBUG 1 + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -49,6 +52,113 @@ extern "C" { CPython refcounting operations on it! */ + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + + + +typedef union _PyStackRef { + uint64_t index; +} _PyStackRef; + +#define Py_TAG_BITS 0 + +PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); +PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref); +PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); + +static const _PyStackRef PyStackRef_NULL = { .index = 0 }; + +#define PyStackRef_None ((_PyStackRef){ .index = 1 } ) +#define PyStackRef_False ((_PyStackRef){ .index = 2 }) +#define PyStackRef_True ((_PyStackRef){ .index = 3 }) + +#define LAST_PREDEFINED_STACKREF_INDEX 3 + +static inline int +PyStackRef_IsNull(_PyStackRef ref) +{ + return ref.index == 0; +} + +static inline int +PyStackRef_IsTrue(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_True; +} + +static inline int +PyStackRef_IsFalse(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_False; +} + +static inline int +PyStackRef_IsNone(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_None; +} + +static inline PyObject * +_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) +{ + _Py_stackref_record_borrow(ref, filename, linenumber); + return _Py_stackref_get_object(ref); +} + +#define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) + +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef ref) +{ + return _Py_stackref_close(ref); +} + +static inline _PyStackRef +_PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) +{ + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline _PyStackRef +_PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) +{ + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline _PyStackRef +_PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber) +{ + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline void +PyStackRef_CLOSE(_PyStackRef ref) +{ + PyObject *obj = _Py_stackref_close(ref); + Py_DECREF(obj); +} + +static inline _PyStackRef +_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) +{ + PyObject *obj = _Py_stackref_get_object(ref); + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref) + +#else + typedef union _PyStackRef { uintptr_t bits; } _PyStackRef; @@ -200,12 +310,15 @@ static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; #define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) #define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) +#endif + // Converts a PyStackRef back to a PyObject *, converting the // stackref to a new reference. #define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) #define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) + #define PyStackRef_CLEAR(op) \ do { \ _PyStackRef *_tmp_op_ptr = &(op); \ diff --git a/Makefile.pre.in b/Makefile.pre.in index 3e880f7800fccf..67acf0fc520087 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -488,6 +488,7 @@ PYTHON_OBJS= \ Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ + Python/stackrefs.o \ Python/structmember.o \ Python/symtable.o \ Python/sysmodule.o \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst new file mode 100644 index 00000000000000..fde12b78ce0444 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst @@ -0,0 +1,4 @@ +Adds stackref debugging when ``Py_STACKREF_DEBUG`` is set. Finds all +double-closes and leaks, logging the origin and last borrow. + +Inspired by HPy's debug mode. https://docs.hpyproject.org/en/latest/debug-mode.html diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 03ed2b9480f8c9..10fd3a982c36f4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -179,9 +179,9 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) if (kind == CO_FAST_FREE) { // The cell was set when the frame was created from // the function's closure. - assert(oldvalue.bits != 0 && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); + assert(!PyStackRef_IsNull(oldvalue) && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); cell = PyStackRef_AsPyObjectBorrow(oldvalue); - } else if (kind & CO_FAST_CELL && oldvalue.bits != 0) { + } else if (kind & CO_FAST_CELL && !PyStackRef_IsNull(oldvalue)) { PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue); if (PyCell_Check(as_obj)) { cell = as_obj; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 30c12dd4dc9205..63cf1978e8abe5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -681,7 +681,7 @@ dummy_func( assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE(left); DEAD(left); - PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); @@ -4509,17 +4509,17 @@ dummy_func( op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); PyObject *result_o; assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4550,7 +4550,10 @@ dummy_func( if (Py_TYPE(func) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); @@ -4568,6 +4571,10 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); result_o = PyObject_Call(func, callargs, kwargs); } PyStackRef_XCLOSE(kwargs_st); diff --git a/Python/ceval.c b/Python/ceval.c index bfdf5687c287db..3cf11b663c571b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -164,7 +164,7 @@ dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) PyErr_Clear(); } // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits)); + printf("<%s at %p>", Py_TYPE(obj)->tp_name, PyStackRef_AsPyObjectBorrow(*ptr)); } printf("]\n"); fflush(stdout); @@ -805,7 +805,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && !defined(Py_STACKREF_DEBUG) /* Set these to invalid but identifiable values for debugging. */ entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry_frame.f_locals = (PyObject*)0xaaa1; @@ -1810,27 +1810,48 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; - PyObject *const *newargs; + _PyStackRef *newargs; + PyObject *const *object_array = NULL; + _PyStackRef stack_array[8]; if (has_dict) { - newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); - if (newargs == NULL) { + object_array = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (object_array == NULL) { PyStackRef_CLOSE(func); goto error; } + size_t total_args = nargs + PyDict_GET_SIZE(kwargs); + assert(sizeof(PyObject *) == sizeof(_PyStackRef)); + newargs = (_PyStackRef *)object_array; + for (size_t i = 0; i < total_args; i++) { + newargs[i] = PyStackRef_FromPyObjectSteal(object_array[i]); + } } else { - newargs = &PyTuple_GET_ITEM(callargs, 0); - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + if (nargs <= 8) { + newargs = stack_array; + } + else { + newargs = PyMem_Malloc(sizeof(_PyStackRef) *nargs); + if (newargs == NULL) { + PyErr_NoMemory(); + PyStackRef_CLOSE(func); + goto error; + } + } + /* We need to create a new reference for all our args since the new frame steals them. */ + for (Py_ssize_t i = 0; i < nargs; i++) { + newargs[i] = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(callargs, i)); } } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, func, locals, - (_PyStackRef const *)newargs, nargs, kwnames, previous + newargs, nargs, kwnames, previous ); if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + _PyStack_UnpackDict_FreeNoDecRef(object_array, kwnames); + } + else if (nargs > 8) { + PyMem_Free((void *)newargs); } /* No need to decref func here because the reference has been stolen by _PyEvalFramePushAndInit. @@ -1850,21 +1871,39 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject* const* args, size_t argcount, PyObject *kwnames) { + size_t total_args = argcount; + if (kwnames) { + total_args += PyTuple_GET_SIZE(kwnames); + } + _PyStackRef stack_array[8]; + _PyStackRef *arguments; + if (total_args <= 8) { + arguments = stack_array; + } + else { + arguments = PyMem_Malloc(sizeof(_PyStackRef) * total_args); + if (arguments == NULL) { + return PyErr_NoMemory(); + } + } /* _PyEvalFramePushAndInit consumes the references * to func, locals and all its arguments */ Py_XINCREF(locals); for (size_t i = 0; i < argcount; i++) { - Py_INCREF(args[i]); + arguments[i] = PyStackRef_FromPyObjectNew(args[i]); } if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_INCREF(args[i+argcount]); + arguments[i+argcount] = PyStackRef_FromPyObjectNew(args[i+argcount]); } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( tstate, PyStackRef_FromPyObjectNew(func), locals, - (_PyStackRef const *)args, argcount, kwnames, NULL); + arguments, argcount, kwnames, NULL); + if (total_args > 8) { + PyMem_Free(arguments); + } if (frame == NULL) { return NULL; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 398816d5f36a1d..f15633fa467376 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -450,7 +450,7 @@ do { \ /* How much scratch space to give stackref to PyObject* conversion. */ #define MAX_STACKREF_SCRATCH 10 -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ @@ -461,7 +461,7 @@ do { \ assert(NAME != NULL); #endif -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ _PyObjectArray_Free(NAME - 1, NAME##_temp); @@ -470,7 +470,7 @@ do { \ (void)(NAME); #endif -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define CONVERSION_FAILED(NAME) ((NAME) == NULL) #else #define CONVERSION_FAILED(NAME) (0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6e752c57cd70f3..22335021faaa6d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -850,7 +850,7 @@ */ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE(left); - PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee5c55a832d460..bed16b60b76a2f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -208,7 +208,7 @@ */ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE(left); - PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); @@ -1675,16 +1675,16 @@ callargs_st = tuple; func_st = func; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); PyObject *result_o; assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-1 - (oparg & 1)] = callargs_st; @@ -1724,19 +1724,22 @@ if (Py_TYPE(func) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -3 - (oparg & 1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -1745,6 +1748,10 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-1 - (oparg & 1)] = callargs_st; if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/pystate.c b/Python/pystate.c index 839413a65a42fb..c546b7c3a9f10e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -19,6 +19,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT +#include "pycore_stackref.h" // Py_STACKREF_DEBUG #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -663,6 +664,23 @@ init_interpreter(PyInterpreterState *interp, /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + interp->next_stackref = 1; + _Py_hashtable_allocator_t alloc = { + .malloc = malloc, + .free = free, + }; + interp->stackref_debug_table = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, + NULL, + &alloc + ); + _Py_stackref_associate(interp, Py_None, PyStackRef_None); + _Py_stackref_associate(interp, Py_False, PyStackRef_False); + _Py_stackref_associate(interp, Py_True, PyStackRef_True); +#endif interp->_initialized = 1; return _PyStatus_OK(); @@ -768,6 +786,11 @@ PyInterpreterState_New(void) return interp; } +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) +extern void +_Py_stackref_report_leaks(PyInterpreterState *interp); +#endif + static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -877,6 +900,12 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + _Py_stackref_report_leaks(interp); + _Py_hashtable_destroy(interp->stackref_debug_table); + interp->stackref_debug_table = NULL; +#endif + if (tstate->interp == interp) { /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? diff --git a/Python/stackrefs.c b/Python/stackrefs.c new file mode 100644 index 00000000000000..9bb46897685570 --- /dev/null +++ b/Python/stackrefs.c @@ -0,0 +1,156 @@ + +#include "Python.h" + +#include "pycore_stackref.h" + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + +#if SIZEOF_VOID_P < 8 +#error "Py_STACKREF_DEBUG requires 64 bit machine" +#endif + +#include "pycore_interp.h" +#include "pycore_hashtable.h" + +typedef struct _table_entry { + PyObject *obj; + const char *classname; + const char *filename; + int linenumber; + const char *filename_borrow; + int linenumber_borrow; +} TableEntry; + +TableEntry * +make_table_entry(PyObject *obj, const char *filename, int linenumber) +{ + TableEntry *result = malloc(sizeof(TableEntry)); + if (result == NULL) { + return NULL; + } + result->obj = obj; + result->classname = Py_TYPE(obj)->tp_name; + result->filename = filename; + result->linenumber = linenumber; + result->filename_borrow = NULL; + return result; +} + +PyObject * +_Py_stackref_get_object(_PyStackRef ref) +{ + if (ref.index == 0) { + return NULL; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(interp != NULL); + if (ref.index >= interp->next_stackref) { + _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + } + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); + } + return entry->obj; +} + +PyObject * +_Py_stackref_close(_PyStackRef ref) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + if (ref.index >= interp->next_stackref) { + _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + } + PyObject *obj; + if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + // Pre-allocated reference to None, False or True -- Do not clear + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + obj = entry->obj; + } + else { + TableEntry *entry = _Py_hashtable_steal(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + } + obj = entry->obj; + free(entry); + } + return obj; +} + +_PyStackRef +_Py_stackref_create(PyObject *obj, const char *filename, int linenumber) +{ + if (obj == NULL) { + Py_FatalError("Cannot create a stackref for NULL"); + } + PyInterpreterState *interp = PyInterpreterState_Get(); + uint64_t new_id = interp->next_stackref++; + TableEntry *entry = make_table_entry(obj, filename, linenumber); + if (entry == NULL) { + Py_FatalError("No memory left for stackref debug table"); + } + if (_Py_hashtable_set(interp->stackref_debug_table, (void *)new_id, entry) < 0) { + Py_FatalError("No memory left for stackref debug table"); + } + return (_PyStackRef){ .index = new_id }; +} + +void +_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber) +{ + if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + return; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + } + entry->filename_borrow = filename; + entry->linenumber_borrow = linenumber; +} + + +void +_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) +{ + assert(interp->next_stackref >= ref.index); + interp->next_stackref = ref.index+1; + TableEntry *entry = make_table_entry(obj, "builtin-object", 0); + if (entry == NULL) { + Py_FatalError("No memory left for stackref debug table"); + } + if (_Py_hashtable_set(interp->stackref_debug_table, (void *)ref.index, (void *)entry) < 0) { + Py_FatalError("No memory left for stackref debug table"); + } +} + + +static int +report_leak(_Py_hashtable_t *ht, const void *key, const void *value, void *leak) +{ + TableEntry *entry = (TableEntry *)value; + if (!_Py_IsStaticImmortal(entry->obj)) { + *(int *)leak = 1; + printf("Stackref leak. Refers to instance of %s at %p. Created at %s:%d", + entry->classname, entry->obj, entry->filename, entry->linenumber); + if (entry->filename_borrow != NULL) { + printf(". Last borrow at %s:%d",entry->filename_borrow, entry->linenumber_borrow); + } + printf("\n"); + } + return 0; +} + +void +_Py_stackref_report_leaks(PyInterpreterState *interp) +{ + int leak = 0; + _Py_hashtable_foreach(interp->stackref_debug_table, report_leak, &leak); + if (leak) { + Py_FatalError("Stackrefs leaked."); + } +} + +#endif From 5a584c8f54bbeceae7ffa501291e29b7ddc8a0b9 Mon Sep 17 00:00:00 2001 From: shallow-beach <96891913+shallow-beach@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:09:56 -0800 Subject: [PATCH 23/33] Python Tutorial typo fix (#128077) --- Doc/tutorial/classes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 492568961d8a51..9d0fab8861d2a9 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -325,7 +325,7 @@ Now what can we do with instance objects? The only operations understood by instance objects are attribute references. There are two kinds of valid attribute names: data attributes and methods. -*data attributes* correspond to "instance variables" in Smalltalk, and to "data +*Data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if ``x`` is the instance of :class:`!MyClass` created above, the following piece of From 3879ca0100942ae15a09ac22889cbe3e46d424eb Mon Sep 17 00:00:00 2001 From: Nico-Posada <102486290+Nico-Posada@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:20:31 -0600 Subject: [PATCH 24/33] gh-128049: Fix type confusion bug with the return value of a custom ExceptionGroup split function (#128079) --- Lib/test/test_except_star.py | 43 +++++++++++++++++++ ...-12-18-14-22-48.gh-issue-128079.SUD5le.rst | 5 +++ Python/ceval.c | 21 ++++++++- 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index c49c6008e08e8c..284907f61213f8 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -952,6 +952,49 @@ def derive(self, excs): self.assertExceptionIsLike(tes, FalsyEG("eg", [TypeError(1)])) self.assertExceptionIsLike(ves, FalsyEG("eg", [ValueError(2)])) + def test_exception_group_subclass_with_bad_split_func(self): + # see gh-128049. + class BadEG1(ExceptionGroup): + def split(self, *args): + return "NOT A 2-TUPLE!" + + class BadEG2(ExceptionGroup): + def split(self, *args): + return ("NOT A 2-TUPLE!",) + + eg_list = [ + (BadEG1("eg", [OSError(123), ValueError(456)]), + r"split must return a tuple, not str"), + (BadEG2("eg", [OSError(123), ValueError(456)]), + r"split must return a 2-tuple, got tuple of size 1") + ] + + for eg_class, msg in eg_list: + with self.assertRaisesRegex(TypeError, msg) as m: + try: + raise eg_class + except* ValueError: + pass + except* OSError: + pass + + self.assertExceptionIsLike(m.exception.__context__, eg_class) + + # we allow tuples of length > 2 for backwards compatibility + class WeirdEG(ExceptionGroup): + def split(self, *args): + return super().split(*args) + ("anything", 123456, None) + + try: + raise WeirdEG("eg", [OSError(123), ValueError(456)]) + except* OSError as e: + oeg = e + except* ValueError as e: + veg = e + + self.assertExceptionIsLike(oeg, WeirdEG("eg", [OSError(123)])) + self.assertExceptionIsLike(veg, WeirdEG("eg", [ValueError(456)])) + class TestExceptStarCleanup(ExceptStarTest): def test_sys_exception_restored(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst new file mode 100644 index 00000000000000..8da4e677f068a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst @@ -0,0 +1,5 @@ +Fix a bug where :keyword:`except* ` does not properly check the +return value of an :exc:`ExceptionGroup`'s :meth:`~BaseExceptionGroup.split` +function, leading to a crash in some cases. Now when :meth:`~BaseExceptionGroup.split` +returns an invalid object, :keyword:`except* ` raises a :exc:`TypeError` +with the original raised :exc:`ExceptionGroup` object chained to it. diff --git a/Python/ceval.c b/Python/ceval.c index 3cf11b663c571b..e92a11b16cec81 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2134,8 +2134,25 @@ _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, if (pair == NULL) { return -1; } - assert(PyTuple_CheckExact(pair)); - assert(PyTuple_GET_SIZE(pair) == 2); + + if (!PyTuple_CheckExact(pair)) { + PyErr_Format(PyExc_TypeError, + "%.200s.split must return a tuple, not %.200s", + Py_TYPE(exc_value)->tp_name, Py_TYPE(pair)->tp_name); + Py_DECREF(pair); + return -1; + } + + // allow tuples of length > 2 for backwards compatibility + if (PyTuple_GET_SIZE(pair) < 2) { + PyErr_Format(PyExc_TypeError, + "%.200s.split must return a 2-tuple, " + "got tuple of size %zd", + Py_TYPE(exc_value)->tp_name, PyTuple_GET_SIZE(pair)); + Py_DECREF(pair); + return -1; + } + *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0)); *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1)); Py_DECREF(pair); From 2a66dd33dfc0b845042da9bb54aaa4e890733f54 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 20 Dec 2024 20:40:58 +0100 Subject: [PATCH 25/33] gh-112328: Make EnumDict usable on its own and document it (GH-123669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafi Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Ethan Furman --- Doc/library/enum.rst | 26 ++++++++++----- Doc/whatsnew/3.13.rst | 6 ++-- Lib/enum.py | 10 +++--- Lib/test/test_enum.py | 33 ++++++++++++++++++- ...-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst | 1 + 5 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 51292a11f507c4..8ca949368db4ff 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -110,6 +110,10 @@ Module Contents ``KEEP`` which allows for more fine-grained control over how invalid values are dealt with in an enumeration. + :class:`EnumDict` + + A subclass of :class:`dict` for use when subclassing :class:`EnumType`. + :class:`auto` Instances are replaced with an appropriate value for Enum members. @@ -149,14 +153,10 @@ Module Contents Return a list of all power-of-two integers contained in a flag. - :class:`EnumDict` - - A subclass of :class:`dict` for use when subclassing :class:`EnumType`. - .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` -.. versionadded:: 3.14 ``EnumDict`` +.. versionadded:: 3.13 ``EnumDict`` --------------- @@ -830,13 +830,23 @@ Data Types .. class:: EnumDict - *EnumDict* is a subclass of :class:`dict` for use when subclassing :class:`EnumType`. + *EnumDict* is a subclass of :class:`dict` that is used as the namespace + for defining enum classes (see :ref:`prepare`). + It is exposed to allow subclasses of :class:`EnumType` with advanced + behavior like having multiple values per member. + It should be called with the name of the enum class being created, otherwise + private names and internal classes will not be handled correctly. + + Note that only the :class:`~collections.abc.MutableMapping` interface + (:meth:`~object.__setitem__` and :meth:`~dict.update`) is overridden. + It may be possible to bypass the checks using other :class:`!dict` + operations like :meth:`|= `. .. attribute:: EnumDict.member_names - Return list of member names. + A list of member names. - .. versionadded:: 3.14 + .. versionadded:: 3.13 --------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a291122aefc2ce..c8e0f94f4246fb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -879,11 +879,13 @@ email (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the :cve:`2023-27043` fix.) + enum ---- -* :class:`~enum.EnumDict` has been made public in :mod:`enum` to better support - subclassing :class:`~enum.EnumType`. +* :class:`~enum.EnumDict` has been made public to better support subclassing + :class:`~enum.EnumType`. + fractions --------- diff --git a/Lib/enum.py b/Lib/enum.py index ccc1da42206474..04443471b40bff 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -342,12 +342,13 @@ class EnumDict(dict): EnumType will use the names found in self._member_names as the enumeration member names. """ - def __init__(self): + def __init__(self, cls_name=None): super().__init__() self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False + self._cls_name = cls_name def __setitem__(self, key, value): """ @@ -358,7 +359,7 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_private(self._cls_name, key): + if self._cls_name is not None and _is_private(self._cls_name, key): # do nothing, name will be a normal attribute pass elif _is_sunder(key): @@ -406,7 +407,7 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - elif _is_internal_class(self._cls_name, value): + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): # do nothing, name will be a normal attribute pass else: @@ -478,8 +479,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = EnumDict() - enum_dict._cls_name = cls + enum_dict = EnumDict(cls) # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b9e13fb8c3585e..8884295b1ab89c 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -14,7 +14,7 @@ from enum import Enum, EnumMeta, IntEnum, StrEnum, EnumType, Flag, IntFlag, unique, auto from enum import STRICT, CONFORM, EJECT, KEEP, _simple_enum, _test_simple_enum from enum import verify, UNIQUE, CONTINUOUS, NAMED_FLAGS, ReprEnum -from enum import member, nonmember, _iter_bits_lsb +from enum import member, nonmember, _iter_bits_lsb, EnumDict from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support @@ -5440,6 +5440,37 @@ def test_convert_repr_and_str(self): self.assertEqual(format(test_type.CONVERT_STRING_TEST_NAME_A), '5') +class TestEnumDict(unittest.TestCase): + def test_enum_dict_in_metaclass(self): + """Test that EnumDict is usable as a class namespace""" + class Meta(type): + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + return EnumDict(cls) + + class MyClass(metaclass=Meta): + a = 1 + + with self.assertRaises(TypeError): + a = 2 # duplicate + + with self.assertRaises(ValueError): + _a_sunder_ = 3 + + def test_enum_dict_standalone(self): + """Test that EnumDict is usable on its own""" + enumdict = EnumDict() + enumdict['a'] = 1 + + with self.assertRaises(TypeError): + enumdict['a'] = 'other value' + + # Only MutableMapping interface is overridden for now. + # If this stops passing, update the documentation. + enumdict |= {'a': 'other value'} + self.assertEqual(enumdict['a'], 'other value') + + # helpers def enum_dir(cls): diff --git a/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst b/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst new file mode 100644 index 00000000000000..96da94a9f211af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst @@ -0,0 +1 @@ +:class:`enum.EnumDict` can now be used without resorting to private API. From a959ea1b0a026ff118975b9b539513b06dde3190 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 22 Dec 2024 01:17:59 +0000 Subject: [PATCH 26/33] GH-127807: pathlib ABCs: remove `PurePathBase._raw_paths` (#127883) Remove the `PurePathBase` initializer, and make `with_segments()` and `__str__()` abstract. This allows us to drop the `_raw_paths` attribute, and also the `Parser.join()` protocol method. --- Lib/pathlib/_abc.py | 40 +++-------- Lib/pathlib/_local.py | 37 ++++++++-- Lib/pathlib/_types.py | 1 - Lib/test/test_pathlib/test_pathlib.py | 25 +++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 85 ++++++++--------------- 5 files changed, 92 insertions(+), 96 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index b4560295300c28..4402efe3a02310 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -44,49 +44,25 @@ class PurePathBase: """Base class for pure path objects. This class *does not* provide several magic methods that are defined in - its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, - __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path - joining methods accept only strings, not os.PathLike objects more broadly. + its subclass PurePath. They are: __init__, __fspath__, __bytes__, + __reduce__, __hash__, __eq__, __lt__, __le__, __gt__, __ge__. """ - __slots__ = ( - # The `_raw_paths` slot stores unjoined string paths. This is set in - # the `__init__()` method. - '_raw_paths', - ) + __slots__ = () parser = posixpath _globber = PathGlobber - def __init__(self, *args): - for arg in args: - if not isinstance(arg, str): - raise TypeError( - f"argument should be a str, not {type(arg).__name__!r}") - self._raw_paths = list(args) - def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. Subclasses may override this method to customize how new path objects are created from methods like `iterdir()`. """ - return type(self)(*pathsegments) + raise NotImplementedError def __str__(self): """Return the string representation of the path, suitable for passing to system calls.""" - paths = self._raw_paths - if len(paths) == 1: - return paths[0] - elif paths: - # Join path segments from the initializer. - path = self.parser.join(*paths) - # Cache the joined path. - paths.clear() - paths.append(path) - return path - else: - paths.append('') - return '' + raise NotImplementedError def as_posix(self): """Return the string representation of the path with forward (/) @@ -234,17 +210,17 @@ def joinpath(self, *pathsegments): paths) or a totally different path (if one of the arguments is anchored). """ - return self.with_segments(*self._raw_paths, *pathsegments) + return self.with_segments(str(self), *pathsegments) def __truediv__(self, key): try: - return self.with_segments(*self._raw_paths, key) + return self.with_segments(str(self), key) except TypeError: return NotImplemented def __rtruediv__(self, key): try: - return self.with_segments(key, *self._raw_paths) + return self.with_segments(key, str(self)) except TypeError: return NotImplemented diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index b933dd512eeb28..7689c10604d4e6 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -77,6 +77,10 @@ class PurePath(PurePathBase): """ __slots__ = ( + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', + # The `_drv`, `_root` and `_tail_cached` slots store parsed and # normalized parts of the path. They are set when any of the `drive`, # `root` or `_tail` properties are accessed for the first time. The @@ -140,9 +144,15 @@ def __init__(self, *args): "object where __fspath__ returns a str, " f"not {type(path).__name__!r}") paths.append(path) - # Avoid calling super().__init__, as an optimisation self._raw_paths = paths + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + def joinpath(self, *pathsegments): """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative @@ -304,14 +314,29 @@ def _parse_pattern(cls, pattern): parts.append('') return parts + @property + def _raw_path(self): + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + path = self.parser.join(*paths) + # Cache the joined path. + paths.clear() + paths.append(path) + return path + else: + paths.append('') + return '' + @property def drive(self): """The drive prefix (letter or UNC path), if any.""" try: return self._drv except AttributeError: - raw_path = PurePathBase.__str__(self) - self._drv, self._root, self._tail_cached = self._parse_path(raw_path) + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) return self._drv @property @@ -320,8 +345,7 @@ def root(self): try: return self._root except AttributeError: - raw_path = PurePathBase.__str__(self) - self._drv, self._root, self._tail_cached = self._parse_path(raw_path) + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) return self._root @property @@ -329,8 +353,7 @@ def _tail(self): try: return self._tail_cached except AttributeError: - raw_path = PurePathBase.__str__(self) - self._drv, self._root, self._tail_cached = self._parse_path(raw_path) + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) return self._tail_cached @property diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py index 60df94d0b46049..baa4a7e2af5b68 100644 --- a/Lib/pathlib/_types.py +++ b/Lib/pathlib/_types.py @@ -14,7 +14,6 @@ class Parser(Protocol): """ sep: str - def join(self, path: str, *paths: str) -> str: ... def split(self, path: str) -> tuple[str, str]: ... def splitdrive(self, path: str) -> tuple[str, str]: ... def splitext(self, path: str) -> tuple[str, str]: ... diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index ac3a3b4f15c10e..ef482c311542fa 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -229,6 +229,31 @@ def test_fspath_common(self): self._check_str(p.__fspath__(), ('a/b',)) self._check_str(os.fspath(p), ('a/b',)) + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def test_bytes_exc_message(self): P = self.cls message = (r"argument should be a str or an os\.PathLike object " diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index e230dd188799a5..9198a0cbc45cee 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -53,7 +53,15 @@ def test_parser(self): class DummyPurePath(PurePathBase): - __slots__ = () + __slots__ = ('_segments',) + + def __init__(self, *segments): + self._segments = segments + + def __str__(self): + if self._segments: + return self.parser.join(*self._segments) + return '' def __eq__(self, other): if not isinstance(other, DummyPurePath): @@ -66,6 +74,9 @@ def __hash__(self): def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) + class DummyPurePathTest(unittest.TestCase): cls = DummyPurePath @@ -97,30 +108,11 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') - def test_bytes(self): - P = self.cls - with self.assertRaises(TypeError): - P(b'a') - with self.assertRaises(TypeError): - P(b'a', 'b') - with self.assertRaises(TypeError): - P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') + def test_fspath_common(self): + self.assertRaises(TypeError, os.fspath, self.cls('')) + + def test_as_bytes_common(self): + self.assertRaises(TypeError, bytes, self.cls('')) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object @@ -1286,36 +1278,6 @@ def test_is_absolute_windows(self): # Tests for the virtual classes. # -class PathBaseTest(PurePathBaseTest): - cls = PathBase - - def test_not_implemented_error(self): - p = self.cls('') - e = NotImplementedError - self.assertRaises(e, p.stat) - self.assertRaises(e, p.exists) - self.assertRaises(e, p.is_dir) - self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.open) - self.assertRaises(e, p.read_bytes) - self.assertRaises(e, p.read_text) - self.assertRaises(e, p.write_bytes, b'foo') - self.assertRaises(e, p.write_text, 'foo') - self.assertRaises(e, p.iterdir) - self.assertRaises(e, lambda: list(p.glob('*'))) - self.assertRaises(e, lambda: list(p.rglob('*'))) - self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.readlink) - self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.mkdir) - - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls('')) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls('')) - class DummyPathIO(io.BytesIO): """ @@ -1342,11 +1304,19 @@ class DummyPath(PathBase): Simple implementation of PathBase that keeps files and directories in memory. """ - __slots__ = () + __slots__ = ('_segments') _files = {} _directories = {} + def __init__(self, *segments): + self._segments = segments + + def __str__(self): + if self._segments: + return self.parser.join(*self._segments) + return '' + def __eq__(self, other): if not isinstance(other, DummyPath): return NotImplemented @@ -1358,6 +1328,9 @@ def __hash__(self): def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) + def stat(self, *, follow_symlinks=True): path = str(self).rstrip('/') if path in self._files: From f5ba74b81979b621e38be70ec3ddad1e7f1365ce Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 22 Dec 2024 01:41:38 +0000 Subject: [PATCH 27/33] GH-127807: pathlib ABCs: remove a few private attributes (#127851) From `PurePathBase` delete `_globber`, `_stack` and `_pattern_str`, and from `PathBase` delete `_glob_selector`. This helps avoid an unpleasant surprise for a users who try to use these names. --- Lib/pathlib/_abc.py | 86 +++++++++++++++++++------------------------ Lib/pathlib/_local.py | 34 +++++++++++++---- 2 files changed, 64 insertions(+), 56 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 4402efe3a02310..b521c757561a99 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -25,6 +25,23 @@ def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' +def _explode_path(path): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = path.parser.split + path = str(path) + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + class PathGlobber(_GlobberBase): """ Class providing shell-style globbing for path objects. @@ -50,7 +67,6 @@ class PurePathBase: __slots__ = () parser = posixpath - _globber = PathGlobber def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -82,7 +98,7 @@ def root(self): @property def anchor(self): """The concatenation of the drive and root, or ''.""" - return self._stack[0] + return _explode_path(self)[0] @property def name(self): @@ -160,8 +176,8 @@ def relative_to(self, other, *, walk_up=False): """ if not isinstance(other, PurePathBase): other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack + anchor0, parts0 = _explode_path(self) + anchor1, parts1 = _explode_path(other) if anchor0 != anchor1: raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -183,8 +199,8 @@ def is_relative_to(self, other): """ if not isinstance(other, PurePathBase): other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack + anchor0, parts0 = _explode_path(self) + anchor1, parts1 = _explode_path(other) if anchor0 != anchor1: return False while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -199,7 +215,7 @@ def is_relative_to(self, other): def parts(self): """An object providing sequence-like access to the components in the filesystem path.""" - anchor, parts = self._stack + anchor, parts = _explode_path(self) if anchor: parts.append(anchor) return tuple(reversed(parts)) @@ -224,23 +240,6 @@ def __rtruediv__(self, key): except TypeError: return NotImplemented - @property - def _stack(self): - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - split = self.parser.split - path = str(self) - parent, name = split(path) - names = [] - while path != parent: - names.append(name) - path = parent - parent, name = split(path) - return path, names - @property def parent(self): """The logical parent of the path.""" @@ -268,11 +267,6 @@ def is_absolute(self): a drive).""" return self.parser.isabs(str(self)) - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - return str(self) - def match(self, path_pattern, *, case_sensitive=None): """ Return True if this path matches the given pattern. If the pattern is @@ -293,7 +287,7 @@ def match(self, path_pattern, *, case_sensitive=None): return False if len(path_parts) > len(pattern_parts) and path_pattern.anchor: return False - globber = self._globber(sep, case_sensitive) + globber = PathGlobber(sep, case_sensitive) for path_part, pattern_part in zip(path_parts, pattern_parts): match = globber.compile(pattern_part) if match(path_part) is None: @@ -309,9 +303,9 @@ def full_match(self, pattern, *, case_sensitive=None): pattern = self.with_segments(pattern) if case_sensitive is None: case_sensitive = _is_case_sensitive(self.parser) - globber = self._globber(pattern.parser.sep, case_sensitive, recursive=True) - match = globber.compile(pattern._pattern_str) - return match(self._pattern_str) is not None + globber = PathGlobber(pattern.parser.sep, case_sensitive, recursive=True) + match = globber.compile(str(pattern)) + return match(str(self)) is not None @@ -463,29 +457,25 @@ def iterdir(self): """ raise NotImplementedError - def _glob_selector(self, parts, case_sensitive, recurse_symlinks): - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.parser) - case_pedantic = False - else: - # The user has expressed a case sensitivity choice, but we don't - # know the case sensitivity of the underlying filesystem, so we - # must use scandir() for everything, including non-wildcard parts. - case_pedantic = True - recursive = True if recurse_symlinks else _no_recurse_symlinks - globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) - return globber.selector(parts) - def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) - anchor, parts = pattern._stack + anchor, parts = _explode_path(pattern) if anchor: raise NotImplementedError("Non-relative patterns are unsupported") - select = self._glob_selector(parts, case_sensitive, recurse_symlinks) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + case_pedantic = False + elif case_sensitive == _is_case_sensitive(self.parser): + case_pedantic = False + else: + case_pedantic = True + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = PathGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts) return select(self) def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 7689c10604d4e6..4897149d7e8a8e 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -5,7 +5,7 @@ import posixpath import sys from errno import EINVAL, EXDEV -from glob import _StringGlobber +from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain from stat import S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence @@ -112,7 +112,6 @@ class PurePath(PurePathBase): '_hash', ) parser = os.path - _globber = _StringGlobber def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing @@ -513,13 +512,22 @@ def as_uri(self): from urllib.parse import quote_from_bytes return prefix + quote_from_bytes(os.fsencode(path)) - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + # The string representation of an empty path is a single dot ('.'). Empty # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str + path = str(self) if self.parts else '' + pattern = str(pattern) if pattern.parts else '' + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) + return globber.compile(pattern)(path) is not None # Subclassing os.PathLike makes isinstance() checks slower, # which in turn makes Path construction slower. Register instead! @@ -749,8 +757,18 @@ def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): kind, including directories) matching the given relative pattern. """ sys.audit("pathlib.Path.glob", self, pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True parts = self._parse_pattern(pattern) - select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts[::-1]) root = str(self) paths = select(root) From 8d9f52a7be5c09c0fd4423943edadaacf6d7f917 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 22 Dec 2024 02:22:08 +0000 Subject: [PATCH 28/33] GH-127807: pathlib ABCs: move private copying methods to dedicated class (#127810) Move 9 private `PathBase` attributes and methods into a new `CopyWorker` class. Change `PathBase.copy` from a method to a `CopyWorker` instance. The methods remain private in the `CopyWorker` class. In future we might make some/all of them public so that user subclasses of `PathBase` can customize the copying process (in particular reading/writing of metadata,) but we'd need to make `PathBase` public first. --- Lib/pathlib/_abc.py | 228 +++++++++++++++++++++++------------------- Lib/pathlib/_local.py | 183 ++++++++++++++++++++++++--------- Lib/pathlib/_os.py | 98 ------------------ 3 files changed, 261 insertions(+), 248 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index b521c757561a99..6acc29ebab2bc5 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -57,6 +57,132 @@ def concat_path(path, text): return path.with_segments(str(path) + text) +class CopyWorker: + """ + Class that implements copying between path objects. An instance of this + class is available from the PathBase.copy property; it's made callable so + that PathBase.copy() can be treated as a method. + + The target path's CopyWorker drives the process from its _create() method. + Files and directories are exchanged by calling methods on the source and + target paths, and metadata is exchanged by calling + source.copy._read_metadata() and target.copy._write_metadata(). + """ + __slots__ = ('_path',) + + def __init__(self, path): + self._path = path + + def __call__(self, target, follow_symlinks=True, dirs_exist_ok=False, + preserve_metadata=False): + """ + Recursively copy this file or directory tree to the given destination. + """ + if not isinstance(target, PathBase): + target = self._path.with_segments(target) + + # Delegate to the target path's CopyWorker object. + return target.copy._create(self._path, follow_symlinks, dirs_exist_ok, preserve_metadata) + + _readable_metakeys = frozenset() + + def _read_metadata(self, metakeys, *, follow_symlinks=True): + """ + Returns path metadata as a dict with string keys. + """ + raise NotImplementedError + + _writable_metakeys = frozenset() + + def _write_metadata(self, metadata, *, follow_symlinks=True): + """ + Sets path metadata from the given dict with string keys. + """ + raise NotImplementedError + + def _create(self, source, follow_symlinks, dirs_exist_ok, preserve_metadata): + self._ensure_distinct_path(source) + if preserve_metadata: + metakeys = self._writable_metakeys & source.copy._readable_metakeys + else: + metakeys = None + if not follow_symlinks and source.is_symlink(): + self._create_symlink(source, metakeys) + elif source.is_dir(): + self._create_dir(source, metakeys, follow_symlinks, dirs_exist_ok) + else: + self._create_file(source, metakeys) + return self._path + + def _create_dir(self, source, metakeys, follow_symlinks, dirs_exist_ok): + """Copy the given directory to our path.""" + children = list(source.iterdir()) + self._path.mkdir(exist_ok=dirs_exist_ok) + for src in children: + dst = self._path.joinpath(src.name) + if not follow_symlinks and src.is_symlink(): + dst.copy._create_symlink(src, metakeys) + elif src.is_dir(): + dst.copy._create_dir(src, metakeys, follow_symlinks, dirs_exist_ok) + else: + dst.copy._create_file(src, metakeys) + if metakeys: + metadata = source.copy._read_metadata(metakeys) + if metadata: + self._write_metadata(metadata) + + def _create_file(self, source, metakeys): + """Copy the given file to our path.""" + self._ensure_different_file(source) + with source.open('rb') as source_f: + try: + with self._path.open('wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not self._path.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {self._path}') from e + raise + if metakeys: + metadata = source.copy._read_metadata(metakeys) + if metadata: + self._write_metadata(metadata) + + def _create_symlink(self, source, metakeys): + """Copy the given symbolic link to our path.""" + self._path.symlink_to(source.readlink()) + if metakeys: + metadata = source.copy._read_metadata(metakeys, follow_symlinks=False) + if metadata: + self._write_metadata(metadata, follow_symlinks=False) + + def _ensure_different_file(self, source): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + pass + + def _ensure_distinct_path(self, source): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if source == self._path: + err = OSError(EINVAL, "Source and target are the same path") + elif source in self._path.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(source) + err.filename2 = str(self._path) + raise err + + class PurePathBase: """Base class for pure path objects. @@ -374,31 +500,6 @@ def is_symlink(self): except (OSError, ValueError): return False - def _ensure_different_file(self, other_path): - """ - Raise OSError(EINVAL) if both paths refer to the same file. - """ - pass - - def _ensure_distinct_path(self, other_path): - """ - Raise OSError(EINVAL) if the other path is within this path. - """ - # Note: there is no straightforward, foolproof algorithm to determine - # if one directory is within another (a particularly perverse example - # would be a single network share mounted in one location via NFS, and - # in another location via CIFS), so we simply checks whether the - # other path is lexically equal to, or within, this path. - if self == other_path: - err = OSError(EINVAL, "Source and target are the same path") - elif self in other_path.parents: - err = OSError(EINVAL, "Source path is a parent of target path") - else: - return - err.filename = str(self) - err.filename2 = str(other_path) - raise err - def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -537,88 +638,13 @@ def symlink_to(self, target, target_is_directory=False): """ raise NotImplementedError - def _symlink_to_target_of(self, link): - """ - Make this path a symlink with the same target as the given link. This - is used by copy(). - """ - self.symlink_to(link.readlink()) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ raise NotImplementedError - # Metadata keys supported by this path type. - _readable_metadata = _writable_metadata = frozenset() - - def _read_metadata(self, keys=None, *, follow_symlinks=True): - """ - Returns path metadata as a dict with string keys. - """ - raise NotImplementedError - - def _write_metadata(self, metadata, *, follow_symlinks=True): - """ - Sets path metadata from the given dict with string keys. - """ - raise NotImplementedError - - def _copy_metadata(self, target, *, follow_symlinks=True): - """ - Copies metadata (permissions, timestamps, etc) from this path to target. - """ - # Metadata types supported by both source and target. - keys = self._readable_metadata & target._writable_metadata - if keys: - metadata = self._read_metadata(keys, follow_symlinks=follow_symlinks) - target._write_metadata(metadata, follow_symlinks=follow_symlinks) - - def _copy_file(self, target): - """ - Copy the contents of this file to the given target. - """ - self._ensure_different_file(target) - with self.open('rb') as source_f: - try: - with target.open('wb') as target_f: - copyfileobj(source_f, target_f) - except IsADirectoryError as e: - if not target.exists(): - # Raise a less confusing exception. - raise FileNotFoundError( - f'Directory does not exist: {target}') from e - else: - raise - - def copy(self, target, *, follow_symlinks=True, dirs_exist_ok=False, - preserve_metadata=False): - """ - Recursively copy this file or directory tree to the given destination. - """ - if not isinstance(target, PathBase): - target = self.with_segments(target) - self._ensure_distinct_path(target) - stack = [(self, target)] - while stack: - src, dst = stack.pop() - if not follow_symlinks and src.is_symlink(): - dst._symlink_to_target_of(src) - if preserve_metadata: - src._copy_metadata(dst, follow_symlinks=False) - elif src.is_dir(): - children = src.iterdir() - dst.mkdir(exist_ok=dirs_exist_ok) - stack.extend((child, dst.joinpath(child.name)) - for child in children) - if preserve_metadata: - src._copy_metadata(dst) - else: - src._copy_file(dst) - if preserve_metadata: - src._copy_metadata(dst) - return target + copy = property(CopyWorker, doc=CopyWorker.__call__.__doc__) def copy_into(self, target_dir, *, follow_symlinks=True, dirs_exist_ok=False, preserve_metadata=False): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 4897149d7e8a8e..915402e6c65b29 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -4,10 +4,10 @@ import os import posixpath import sys -from errno import EINVAL, EXDEV +from errno import * from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain -from stat import S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from stat import S_IMODE, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence try: @@ -19,9 +19,8 @@ except ImportError: grp = None -from pathlib._os import (copyfile, file_metadata_keys, read_file_metadata, - write_file_metadata) -from pathlib._abc import PurePathBase, PathBase +from pathlib._os import copyfile +from pathlib._abc import CopyWorker, PurePathBase, PathBase __all__ = [ @@ -66,6 +65,131 @@ def __repr__(self): return "<{}.parents>".format(type(self._path).__name__) +class _LocalCopyWorker(CopyWorker): + """This object implements the Path.copy callable. Don't try to construct + it yourself.""" + __slots__ = () + + _readable_metakeys = {'mode', 'times_ns'} + if hasattr(os.stat_result, 'st_flags'): + _readable_metakeys.add('flags') + if hasattr(os, 'listxattr'): + _readable_metakeys.add('xattrs') + _readable_metakeys = _writable_metakeys = frozenset(_readable_metakeys) + + def _read_metadata(self, metakeys, *, follow_symlinks=True): + metadata = {} + if 'mode' in metakeys or 'times_ns' in metakeys or 'flags' in metakeys: + st = self._path.stat(follow_symlinks=follow_symlinks) + if 'mode' in metakeys: + metadata['mode'] = S_IMODE(st.st_mode) + if 'times_ns' in metakeys: + metadata['times_ns'] = st.st_atime_ns, st.st_mtime_ns + if 'flags' in metakeys: + metadata['flags'] = st.st_flags + if 'xattrs' in metakeys: + try: + metadata['xattrs'] = [ + (attr, os.getxattr(self._path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(self._path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + return metadata + + def _write_metadata(self, metadata, *, follow_symlinks=True): + def _nop(*args, ns=None, follow_symlinks=None): + pass + + if follow_symlinks: + # use the real function if it exists + def lookup(name): + return getattr(os, name, _nop) + else: + # use the real function only if it exists + # *and* it supports follow_symlinks + def lookup(name): + fn = getattr(os, name, _nop) + if fn in os.supports_follow_symlinks: + return fn + return _nop + + times_ns = metadata.get('times_ns') + if times_ns is not None: + lookup("utime")(self._path, ns=times_ns, follow_symlinks=follow_symlinks) + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + xattrs = metadata.get('xattrs') + if xattrs is not None: + for attr, value in xattrs: + try: + os.setxattr(self._path, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + mode = metadata.get('mode') + if mode is not None: + try: + lookup("chmod")(self._path, mode, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + flags = metadata.get('flags') + if flags is not None: + try: + lookup("chflags")(self._path, flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise + + if copyfile: + # Use fast OS routine for local file copying where available. + def _create_file(self, source, metakeys): + """Copy the given file to the given target.""" + try: + source = os.fspath(source) + except TypeError: + if not isinstance(source, PathBase): + raise + super()._create_file(source, metakeys) + else: + copyfile(source, os.fspath(self._path)) + + if os.name == 'nt': + # Windows: symlink target might not exist yet if we're copying several + # files, so ensure we pass is_dir to os.symlink(). + def _create_symlink(self, source, metakeys): + """Copy the given symlink to the given target.""" + self._path.symlink_to(source.readlink(), source.is_dir()) + if metakeys: + metadata = source.copy._read_metadata(metakeys, follow_symlinks=False) + if metadata: + self._write_metadata(metadata, follow_symlinks=False) + + def _ensure_different_file(self, source): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + if not self._path.samefile(source): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(source) + err.filename2 = str(self._path) + raise err + + class PurePath(PurePathBase): """Base class for manipulating paths without I/O. @@ -678,20 +802,6 @@ def samefile(self, other_path): return (st.st_ino == other_st.st_ino and st.st_dev == other_st.st_dev) - def _ensure_different_file(self, other_path): - """ - Raise OSError(EINVAL) if both paths refer to the same file. - """ - try: - if not self.samefile(other_path): - return - except (OSError, ValueError): - return - err = OSError(EINVAL, "Source and target are the same file") - err.filename = str(self) - err.filename2 = str(other_path) - raise err - def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -932,24 +1042,6 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): if not exist_ok or not self.is_dir(): raise - _readable_metadata = _writable_metadata = file_metadata_keys - _read_metadata = read_file_metadata - _write_metadata = write_file_metadata - - if copyfile: - def _copy_file(self, target): - """ - Copy the contents of this file to the given target. - """ - try: - target = os.fspath(target) - except TypeError: - if not isinstance(target, PathBase): - raise - PathBase._copy_file(self, target) - else: - copyfile(os.fspath(self), target) - def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). @@ -1019,16 +1111,17 @@ def replace(self, target): os.replace(self, target) return self.with_segments(target) + copy = property(_LocalCopyWorker, doc=_LocalCopyWorker.__call__.__doc__) + def move(self, target): """ Recursively move this file or directory tree to the given destination. """ - self._ensure_different_file(target) + if not isinstance(target, PathBase): + target = self.with_segments(target) + target.copy._ensure_different_file(self) try: return self.replace(target) - except TypeError: - if not isinstance(target, PathBase): - raise except OSError as err: if err.errno != EXDEV: raise @@ -1051,14 +1144,6 @@ def symlink_to(self, target, target_is_directory=False): f = f"{type(self).__name__}.symlink_to()" raise UnsupportedOperation(f"{f} is unsupported on this system") - if os.name == 'nt': - def _symlink_to_target_of(self, link): - """ - Make this path a symlink with the same target as the given link. - This is used by copy(). - """ - self.symlink_to(link.readlink(), link.is_dir()) - if hasattr(os, "link"): def hardlink_to(self, target): """ diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py index 642b3a57c59a1d..57bcaf3d680138 100644 --- a/Lib/pathlib/_os.py +++ b/Lib/pathlib/_os.py @@ -4,7 +4,6 @@ from errno import * import os -import stat import sys try: import fcntl @@ -163,100 +162,3 @@ def copyfileobj(source_f, target_f): write_target = target_f.write while buf := read_source(1024 * 1024): write_target(buf) - - -# Kinds of metadata supported by the operating system. -file_metadata_keys = {'mode', 'times_ns'} -if hasattr(os.stat_result, 'st_flags'): - file_metadata_keys.add('flags') -if hasattr(os, 'listxattr'): - file_metadata_keys.add('xattrs') -file_metadata_keys = frozenset(file_metadata_keys) - - -def read_file_metadata(path, keys=None, *, follow_symlinks=True): - """ - Returns local path metadata as a dict with string keys. - """ - if keys is None: - keys = file_metadata_keys - assert keys.issubset(file_metadata_keys) - result = {} - for key in keys: - if key == 'xattrs': - try: - result['xattrs'] = [ - (attr, os.getxattr(path, attr, follow_symlinks=follow_symlinks)) - for attr in os.listxattr(path, follow_symlinks=follow_symlinks)] - except OSError as err: - if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): - raise - continue - st = os.stat(path, follow_symlinks=follow_symlinks) - if key == 'mode': - result['mode'] = stat.S_IMODE(st.st_mode) - elif key == 'times_ns': - result['times_ns'] = st.st_atime_ns, st.st_mtime_ns - elif key == 'flags': - result['flags'] = st.st_flags - return result - - -def write_file_metadata(path, metadata, *, follow_symlinks=True): - """ - Sets local path metadata from the given dict with string keys. - """ - assert frozenset(metadata.keys()).issubset(file_metadata_keys) - - def _nop(*args, ns=None, follow_symlinks=None): - pass - - if follow_symlinks: - # use the real function if it exists - def lookup(name): - return getattr(os, name, _nop) - else: - # use the real function only if it exists - # *and* it supports follow_symlinks - def lookup(name): - fn = getattr(os, name, _nop) - if fn in os.supports_follow_symlinks: - return fn - return _nop - - times_ns = metadata.get('times_ns') - if times_ns is not None: - lookup("utime")(path, ns=times_ns, follow_symlinks=follow_symlinks) - # We must copy extended attributes before the file is (potentially) - # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. - xattrs = metadata.get('xattrs') - if xattrs is not None: - for attr, value in xattrs: - try: - os.setxattr(path, attr, value, follow_symlinks=follow_symlinks) - except OSError as e: - if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): - raise - mode = metadata.get('mode') - if mode is not None: - try: - lookup("chmod")(path, mode, follow_symlinks=follow_symlinks) - except NotImplementedError: - # if we got a NotImplementedError, it's because - # * follow_symlinks=False, - # * lchown() is unavailable, and - # * either - # * fchownat() is unavailable or - # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. - # (it returned ENOSUP.) - # therefore we're out of options--we simply cannot chown the - # symlink. give up, suppress the error. - # (which is what shutil always did in this circumstance.) - pass - flags = metadata.get('flags') - if flags is not None: - try: - lookup("chflags")(path, flags, follow_symlinks=follow_symlinks) - except OSError as why: - if why.errno not in (EOPNOTSUPP, ENOTSUP): - raise From 228f275737615cc9be713a8c3f9325b359bf8aec Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 22 Dec 2024 01:47:41 -0800 Subject: [PATCH 29/33] gh-126664: revert: Use `else` instead of `finally` in docs explaining "with" (#128169) Revert "gh-126664: Use `else` instead of `finally` in "The with statement" documentation. (GH-126665)" This reverts commit 25257d61cfccc3b4189f96390a5c4db73fd5302c. --- Doc/reference/compound_stmts.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e73ce44270b082..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -534,15 +534,18 @@ is semantically equivalent to:: enter = type(manager).__enter__ exit = type(manager).__exit__ value = enter(manager) + hit_except = False try: TARGET = value SUITE except: + hit_except = True if not exit(manager, *sys.exc_info()): raise - else: - exit(manager, None, None, None) + finally: + if not hit_except: + exit(manager, None, None, None) With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: From b66a4ad9fc32b63da2ba10db24cbc8f4e29f781a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 22 Dec 2024 12:46:02 +0000 Subject: [PATCH 30/33] gh-127949: fix resource warnings in `test_tasks.py` (#128172) --- Lib/test/test_asyncio/test_tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5b8979a8bbd13a..7d6d0564a9a9db 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2698,17 +2698,17 @@ def __str__(self): initial_refcount = sys.getrefcount(obj) coro = coroutine_function() - loop = asyncio.new_event_loop() - task = asyncio.Task.__new__(asyncio.Task) + with contextlib.closing(asyncio.EventLoop()) as loop: + task = asyncio.Task.__new__(asyncio.Task) - for _ in range(5): - with self.assertRaisesRegex(RuntimeError, 'break'): - task.__init__(coro, loop=loop, context=obj, name=Break()) + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) - coro.close() - del task + coro.close() + del task - self.assertEqual(sys.getrefcount(obj), initial_refcount) + self.assertEqual(sys.getrefcount(obj), initial_refcount) def add_subclass_tests(cls): From f420bdd29fbc1a97ad20d88075c38c937c1f8479 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sun, 22 Dec 2024 17:34:16 +0100 Subject: [PATCH 31/33] gh-119786: Fix typos in `InternalDocs/interpreter.md` (#128174) --- InternalDocs/interpreter.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index fa4a54fdc54fac..52702792c6cb7b 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -20,7 +20,7 @@ When the interpreter's [`PyEval_EvalCode()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) function is called to execute a `CodeObject`, it constructs a [`Frame`](frames.md) and calls [`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) -to execute the code object in this frame. The frame hold the dynamic state of the +to execute the code object in this frame. The frame holds the dynamic state of the `CodeObject`'s execution, including the instruction pointer, the globals and builtins. It also has a reference to the `CodeObject` itself. @@ -153,9 +153,9 @@ More information about the use of inline caches can be found in Most instructions read or write some data in the form of object references (`PyObject *`). The CPython bytecode interpreter is a stack machine, meaning that its instructions operate by pushing data onto and popping it off the stack. -The stack is forms part of the frame for the code object. Its maximum depth is calculated +The stack forms part of the frame for the code object. Its maximum depth is calculated by the compiler and stored in the `co_stacksize` field of the code object, so that the -stack can be pre-allocated is a contiguous array of `PyObject*` pointers, when the frame +stack can be pre-allocated as a contiguous array of `PyObject*` pointers, when the frame is created. The stack effects of each instruction are also exposed through the @@ -462,7 +462,7 @@ set of values that allows them to: 2. Perform the operation quickly. This requires that the set of values is chosen such that membership can be -tested quickly and that membership is sufficient to allow the operation to +tested quickly and that membership is sufficient to allow the operation to be performed quickly. For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` From 9d3a8f494985e8bbef698c467099370e233fcbd4 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sun, 22 Dec 2024 13:01:45 -0600 Subject: [PATCH 32/33] gh-100384: Error on `unguarded-availability` in macOS builds (#128155) Generate a build error on ``unguarded-availability`` in portable macOS builds (i.e. using MACOSX_DEPLOYMENT_TARGET), preventing invalid use of symbols that are not available in older versions of the OS. --- .github/workflows/reusable-macos.yml | 1 + ...-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst | 2 + configure | 41 +++++++++++++++++++ configure.ac | 7 ++++ 4 files changed, 51 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 6fa389b2d66e5e..cdbe05e09fb8e7 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -45,6 +45,7 @@ jobs: brew link --overwrite tcl-tk@8 - name: Configure CPython run: | + MACOSX_DEPLOYMENT_TARGET=10.15 \ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ diff --git a/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst new file mode 100644 index 00000000000000..75c19fe3d8cef9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst @@ -0,0 +1,2 @@ +Error on ``unguarded-availability`` in macOS builds, preventing invalid +use of symbols that are not available in older versions of the OS. diff --git a/configure b/configure index e59c7046305d46..a697bc1f87b012 100755 --- a/configure +++ b/configure @@ -10406,6 +10406,47 @@ printf %s "checking which compiler should be used... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 printf "%s\n" "$CC" >&6; } + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wunguarded-availability" >&5 +printf %s "checking whether C compiler accepts -Wunguarded-availability... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wunguarded_availability+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wunguarded-availability" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wunguarded_availability=yes +else $as_nop + ax_cv_check_cflags__Werror__Wunguarded_availability=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wunguarded_availability" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wunguarded_availability" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wunguarded_availability" = xyes +then : + as_fn_append CFLAGS_NODIST " -Werror=unguarded-availability" +else $as_nop + : +fi + + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" then diff --git a/configure.ac b/configure.ac index 074e2ce3dd3024..ebc15503f069cc 100644 --- a/configure.ac +++ b/configure.ac @@ -2603,6 +2603,13 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ esac AC_MSG_RESULT([$CC]) + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + AX_CHECK_COMPILE_FLAG([-Wunguarded-availability], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -Werror=unguarded-availability"])], + [], + [-Werror]) + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" then From 831b6de6d725c697f2f61fd35c4448cd8a9354ff Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Mon, 23 Dec 2024 14:17:19 +1000 Subject: [PATCH 33/33] gh-126180: Remove getopt and optparse deprecation notices (GH-126227) * Remove getopt and optparse deprecation notices * Add new docs sections for command line app helper libraries * Add guidance on choosing a CLI parsing library to the optparse docs * Link to the new guidance from the argparse and getopt docs * Reword intro in docs section for superseded stdlib modules * Reframe the optparse->argparse guide as a migration guide rather than as an upgrade guide --------- Co-authored-by: Serhiy Storchaka --- Doc/howto/argparse-optparse.rst | 36 +++-- Doc/howto/argparse.rst | 15 +- Doc/library/allos.rst | 5 - Doc/library/argparse.rst | 12 ++ Doc/library/cmdlinelibs.rst | 21 +++ Doc/library/filesys.rst | 1 - Doc/library/getopt.rst | 54 +++++-- Doc/library/index.rst | 1 + Doc/library/optparse.rst | 139 ++++++++++++++++-- Doc/library/superseded.rst | 17 ++- Doc/whatsnew/3.13.rst | 24 ++- ...-10-31-14-31-36.gh-issue-126225.vTxGXm.rst | 6 + 12 files changed, 266 insertions(+), 65 deletions(-) create mode 100644 Doc/library/cmdlinelibs.rst create mode 100644 Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst index cef2d893b28a62..b684619885b4c7 100644 --- a/Doc/howto/argparse-optparse.rst +++ b/Doc/howto/argparse-optparse.rst @@ -1,20 +1,14 @@ .. currentmodule:: argparse .. _upgrading-optparse-code: +.. _migrating-optparse-code: -========================== -Upgrading optparse code -========================== +============================================ +Migrating ``optparse`` code to ``argparse`` +============================================ -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the :mod:`optparse` -module in a number of ways including: +The :mod:`argparse` module offers several higher level features not natively +provided by the :mod:`optparse` module, including: * Handling positional arguments. * Supporting subcommands. @@ -23,7 +17,23 @@ module in a number of ways including: * Producing more informative usage messages. * Providing a much simpler interface for custom ``type`` and ``action``. -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: +Originally, the :mod:`argparse` module attempted to maintain compatibility +with :mod:`optparse`. However, the fundamental design differences between +supporting declarative command line option processing (while leaving positional +argument processing to application code), and supporting both named options +and positional arguments in the declarative interface mean that the +API has diverged from that of ``optparse`` over time. + +As described in :ref:`choosing-an-argument-parser`, applications that are +currently using :mod:`optparse` and are happy with the way it works can +just continue to use ``optparse``. + +Application developers that are considering migrating should also review +the list of intrinsic behavioural differences described in that section +before deciding whether or not migration is desirable. + +For applications that do choose to migrate from :mod:`optparse` to :mod:`argparse`, +the following suggestions should be helpful: * Replace all :meth:`optparse.OptionParser.add_option` calls with :meth:`ArgumentParser.add_argument` calls. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 1efbee64d60bb3..902c50de00803c 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -13,11 +13,16 @@ recommended command-line parsing module in the Python standard library. .. note:: - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for ``getopt()`` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. + The standard library includes two other libraries directly related + to command-line parameter processing: the lower level :mod:`optparse` + module (which may require more code to configure for a given application, + but also allows an application to request behaviors that ``argparse`` + doesn't support), and the very low level :mod:`getopt` (which specifically + serves as an equivalent to the :c:func:`!getopt` family of functions + available to C programmers). + While neither of those modules is covered directly in this guide, many of + the core concepts in ``argparse`` first originated in ``optparse``, so + some aspects of this tutorial will also be relevant to ``optparse`` users. Concepts diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index 0223c1054ea5d8..1aed340b2527ac 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -15,14 +15,9 @@ but they are available on most other systems as well. Here's an overview: os.rst io.rst time.rst - argparse.rst logging.rst logging.config.rst logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst platform.rst errno.rst ctypes.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index da4071dee34b8c..8d0116d8c060b8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -11,6 +11,18 @@ **Source code:** :source:`Lib/argparse.py` +.. note:: + + While :mod:`argparse` is the default recommended standard library module + for implementing basic command line applications, authors with more + exacting requirements for exactly how their command line applications + behave may find it doesn't provide the necessary level of control. + Refer to :ref:`choosing-an-argument-parser` for alternatives to + consider when ``argparse`` doesn't support behaviors that the application + requires (such as entirely disabling support for interspersed options and + positional arguments, or accepting option parameter values that start + with ``-`` even when they correspond to another defined option). + -------------- .. sidebar:: Tutorial diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst new file mode 100644 index 00000000000000..085d31af7bca1f --- /dev/null +++ b/Doc/library/cmdlinelibs.rst @@ -0,0 +1,21 @@ +.. _cmdlinelibs: + +******************************** +Command Line Interface Libraries +******************************** + +The modules described in this chapter assist with implementing +command line and terminal interfaces for applications. + +Here's an overview: + +.. toctree:: + :maxdepth: 1 + + argparse.rst + optparse.rst + getpass.rst + fileinput.rst + curses.rst + curses.ascii.rst + curses.panel.rst diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst index 0ccf2b7bf59a0f..f1ea4761af7cb1 100644 --- a/Doc/library/filesys.rst +++ b/Doc/library/filesys.rst @@ -14,7 +14,6 @@ in this chapter is: pathlib.rst os.path.rst - fileinput.rst stat.rst filecmp.rst tempfile.rst diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 891885d3afbf7a..5c63009e22d58c 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -7,18 +7,13 @@ **Source code:** :source:`Lib/getopt.py` -.. deprecated:: 3.13 - The :mod:`getopt` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - .. note:: - The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`!getopt` function. Users who - are unfamiliar with the C :c:func:`!getopt` function or who would like to write - less code and get better help and error messages should consider using the - :mod:`argparse` module instead. + This module is considered feature complete. A more declarative and + extensible alternative to this API is provided in the :mod:`optparse` + module. Further functional enhancements for command line parameter + processing are provided either as third party modules on PyPI, + or else as features in the :mod:`argparse` module. -------------- @@ -28,6 +23,13 @@ the special meanings of arguments of the form '``-``' and '``--``'). Long options similar to those supported by GNU software may be used as well via an optional third argument. +Users who are unfamiliar with the Unix :c:func:`!getopt` function should consider +using the :mod:`argparse` module instead. Users who are familiar with the Unix +:c:func:`!getopt` function, but would like to get equivalent behavior while +writing less code and getting better help and error messages should consider +using the :mod:`optparse` module. See :ref:`choosing-an-argument-parser` for +additional details. + This module provides two functions and an exception: @@ -194,13 +196,27 @@ In a script, typical usage is something like this: output = a else: assert False, "unhandled option" - # ... + process(args, output=output, verbose=verbose) if __name__ == "__main__": main() Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module: +and more informative help and error messages by using the :mod:`optparse` module: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +A roughly equivalent command line interface for this case can also be +produced by using the :mod:`argparse` module: .. testcode:: @@ -210,12 +226,18 @@ and more informative help and error messages by using the :mod:`argparse` module parser = argparse.ArgumentParser() parser.add_argument('-o', '--output') parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') args = parser.parse_args() - # ... do something with args.output ... - # ... do something with args.verbose .. + process(args.rest, output=args.output, verbose=args.verbose) + +See :ref:`choosing-an-argument-parser` for details on how the ``argparse`` +version of this code differs in behaviour from the ``optparse`` (and +``getopt``) version. .. seealso:: - Module :mod:`argparse` - Alternative command line option and argument parsing library. + Module :mod:`optparse` + Declarative command line option parsing. + Module :mod:`argparse` + More opinionated command line option and argument parsing library. diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 951fbcf13fbb13..44b218948d07e1 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -55,6 +55,7 @@ the `Python Package Index `_. fileformats.rst crypto.rst allos.rst + cmdlinelibs.rst concurrency.rst ipc.rst netdata.rst diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 74a49a8fb33666..ff327cf9162a8c 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -3,25 +3,135 @@ .. module:: optparse :synopsis: Command-line option parsing library. - :deprecated: .. moduleauthor:: Greg Ward .. sectionauthor:: Greg Ward **Source code:** :source:`Lib/optparse.py` -.. deprecated:: 3.2 - The :mod:`optparse` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - -------------- +.. _choosing-an-argument-parser: + +Choosing an argument parsing library +------------------------------------ + +The standard library includes three argument parsing libraries: + +* :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. + Included in the standard library since before the initial Python 1.0 release. +* :mod:`optparse`: a declarative replacement for ``getopt`` that + provides equivalent functionality without requiring each application + to implement its own procedural option parsing logic. Included + in the standard library since the Python 2.3 release. +* :mod:`argparse`: a more opinionated alternative to ``optparse`` that + provides more functionality by default, at the expense of reduced application + flexibility in controlling exactly how arguments are processed. Included in + the standard library since the Python 2.7 and Python 3.2 releases. + +In the absence of more specific argument parsing design constraints, :mod:`argparse` +is the recommended choice for implementing command line applications, as it offers +the highest level of baseline functionality with the least application level code. + +:mod:`getopt` is retained almost entirely for backwards compatibility reasons. +However, it also serves a niche use case as a tool for prototyping and testing +command line argument handling in ``getopt``-based C applications. + +:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +following cases: + +* an application is already using :mod:`optparse` and doesn't want to risk the + subtle behavioural changes that may arise when migrating to :mod:`argparse` +* the application requires additional control over the way options and + positional parameters are interleaved on the command line (including + the ability to disable the interleaving feature completely) +* the application requires additional control over the incremental parsing + of command line elements (while ``argparse`` does support this, the + exact way it works in practice is undesirable for some use cases) +* the application requires additional control over the handling of options + which accept parameter values that may start with ``-`` (such as delegated + options to be passed to invoked subprocesses) +* the application requires some other command line parameter processing + behavior which ``argparse`` does not support, but which can be implemented + in terms of the lower level interface offered by ``optparse`` + +These considerations also mean that :mod:`optparse` is likely to provide a +better foundation for library authors writing third party command line +argument processing libraries. + +As a concrete example, consider the following two command line argument +parsing configurations, the first using ``optparse``, and the second +using ``argparse``: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +.. testcode:: + + import argparse + + if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output') + parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') + args = parser.parse_args() + process(args.rest, output=args.output, verbose=args.verbose) + +The most obvious difference is that in the ``optparse`` version, the non-option +arguments are processed separately by the application after the option processing +is complete. In the ``argparse`` version, positional arguments are declared and +processed in the same way as the named options. + +However, the ``argparse`` version will also handle some parameter combination +differently from the way the ``optparse`` version would handle them. +For example (amongst other differences): + +* supplying ``-o -v`` gives ``output="-v"`` and ``verbose=False`` + when using ``optparse``, but a usage error with ``argparse`` + (complaining that no value has been supplied for ``-o/--output``, + since ``-v`` is interpreted as meaning the verbosity flag) +* similarly, supplying ``-o --`` gives ``output="--"`` and ``args=()`` + when using ``optparse``, but a usage error with ``argparse`` + (also complaining that no value has been supplied for ``-o/--output``, + since ``--`` is interpreted as terminating the option processing + and treating all remaining values as positional arguments) +* supplying ``-o=foo`` gives ``output="=foo"`` when using ``optparse``, + but gives ``output="foo"`` with ``argparse`` (since ``=`` is special + cased as an alternative separator for option parameter values) + +Whether these differing behaviors in the ``argparse`` version are +considered desirable or a problem will depend on the specific command line +application use case. + +.. seealso:: + + :pypi:`click` is a third party argument processing library (originally + based on ``optparse``), which allows command line applications to be + developed as a set of decorated command implementation functions. + + Other third party libraries, such as :pypi:`typer` or :pypi:`msgspec-click`, + allow command line interfaces to be specified in ways that more effectively + integrate with static checking of Python type annotations. + + +Introduction +------------ + :mod:`optparse` is a more convenient, flexible, and powerful library for parsing -command-line options than the old :mod:`getopt` module. :mod:`optparse` uses a -more declarative style of command-line parsing: you create an instance of -:class:`OptionParser`, populate it with options, and parse the command -line. :mod:`optparse` allows users to specify options in the conventional +command-line options than the minimalist :mod:`getopt` module. +:mod:`optparse` uses a more declarative style of command-line parsing: +you create an instance of :class:`OptionParser`, +populate it with options, and parse the command line. +:mod:`optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. Here's an example of using :mod:`optparse` in a simple script:: @@ -82,10 +192,11 @@ Background ---------- :mod:`optparse` was explicitly designed to encourage the creation of programs -with straightforward, conventional command-line interfaces. To that end, it -supports only the most common command-line syntax and semantics conventionally -used under Unix. If you are unfamiliar with these conventions, read this -section to acquaint yourself with them. +with straightforward command-line interfaces that follow the conventions +established by the :c:func:`!getopt` family of functions available to C developers. +To that end, it supports only the most common command-line syntax and semantics +conventionally used under Unix. If you are unfamiliar with these conventions, +reading this section will allow you to acquaint yourself with them. .. _optparse-terminology: diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index 17bfa66f043302..d120c6acf621e3 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -4,12 +4,23 @@ Superseded Modules ****************** -The modules described in this chapter are deprecated or :term:`soft deprecated` and only kept for -backwards compatibility. They have been superseded by other modules. +The modules described in this chapter have been superseded by other modules +for most use cases, and are retained primarily to preserve backwards compatibility. +Modules may appear in this chapter because they only cover a limited subset of +a problem space, and a more generally applicable solution is available elsewhere +in the standard library (for example, :mod:`getopt` covers the very specific +task of "mimic the C :c:func:`!getopt` API in Python", rather than the broader +command line option parsing and argument parsing capabilities offered by +:mod:`optparse` and :mod:`argparse`). + +Alternatively, modules may appear in this chapter because they are deprecated +outright, and awaiting removal in a future release, or they are +:term:`soft deprecated` and their use is actively discouraged in new projects. +With the removal of various obsolete modules through :pep:`594`, there are +currently no modules in this latter category. .. toctree:: :maxdepth: 1 getopt.rst - optparse.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c8e0f94f4246fb..6a0e483bd895d6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1648,6 +1648,22 @@ opcode (Contributed by Irit Katriel in :gh:`105481`.) +optparse +-------- + +* This module is no longer considered :term:`soft deprecated`. + While :mod:`argparse` remains preferred for new projects that + aren't using a third party command line argument processing + library, there are aspects of the way ``argparse`` works that + mean the lower level ``optparse`` module may provide a better + foundation for *writing* argument processing libraries, and + for implementing command line applications which adhere more + strictly than ``argparse`` does to various Unix command line + processing conventions that originate in the behaviour of the + C :c:func:`!getopt` function . + (Contributed by Alyssa Coghlan and Serhiy Storchaka in :gh:`126180`.) + + pathlib ------- @@ -1787,14 +1803,6 @@ New Deprecations Check membership in :data:`~dis.hasarg` instead. (Contributed by Irit Katriel in :gh:`109319`.) -* :mod:`getopt` and :mod:`optparse`: - - * Both modules are now :term:`soft deprecated`, - with :mod:`argparse` preferred for new projects. - This is a new soft-deprecation for the :mod:`!getopt` module, - whereas the :mod:`!optparse` module was already *de facto* soft deprecated. - (Contributed by Victor Stinner in :gh:`106535`.) - * :mod:`gettext`: * Deprecate non-integer numbers as arguments to functions and methods diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst new file mode 100644 index 00000000000000..13a1f213c7a58e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst @@ -0,0 +1,6 @@ +:mod:`getopt` and :mod:`optparse` are no longer marked as deprecated. +There are legitimate reasons to use one of these modules in preference to +:mod:`argparse`, and none of these modules are at risk of being removed +from the standard library. Of the three, ``argparse`` remains the +recommended default choice, *unless* one of the concerns noted at the top of +the ``optparse`` module documentation applies.