GH-140683: JIT: Improve machine code for loading smaller constants on AArch64. (GH...

* Use movz and movk instructions for loading 16 and 32 bit operands and oparg.
* Loading of 64 bit operands is unchanged.

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 9eac58af3e26227750e42ad4fb9db2251719cc67..f64a7778121c71010244409f7d2559f3c9ecc4d0 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -810,7 +810,7 @@ dummy_func(
             assert(next_instr->op.code == STORE_FAST);
             next_oparg = next_instr->op.arg;
         #else
-            next_oparg = (int)CURRENT_OPERAND0();
+            next_oparg = (int)CURRENT_OPERAND0_16();
         #endif
             _PyStackRef *target_local = &GETLOCAL(next_oparg);
             assert(PyUnicode_CheckExact(left_o));

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index d4ea1162eba4449189a1b0a28f3bd055773cce18..26312d0ea5aa5cd5f4761a595b6658c9126ec835 100644 (file)
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -450,8 +450,12 @@ do {                                                   \
     } while (0)
 
 #define CURRENT_OPARG()    (next_uop[-1].oparg)
-#define CURRENT_OPERAND0() (next_uop[-1].operand0)
-#define CURRENT_OPERAND1() (next_uop[-1].operand1)
+#define CURRENT_OPERAND0_64() (next_uop[-1].operand0)
+#define CURRENT_OPERAND1_64() (next_uop[-1].operand1)
+#define CURRENT_OPERAND0_32() (next_uop[-1].operand0)
+#define CURRENT_OPERAND1_32() (next_uop[-1].operand1)
+#define CURRENT_OPERAND0_16() (next_uop[-1].operand0)
+#define CURRENT_OPERAND1_16() (next_uop[-1].operand1)
 #define CURRENT_TARGET()   (next_uop[-1].target)
 
 #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index a64fd5b28f1d5770f477d6c09ee94b73b1fe1d1d..24f343c994038bd46781a765c261a2171d36bb8c 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -4645,7 +4645,7 @@
             assert(next_instr->op.code == STORE_FAST);
             next_oparg = next_instr->op.arg;
             #else
-            next_oparg = (int)CURRENT_OPERAND0();
+            next_oparg = (int)CURRENT_OPERAND0_16();
             #endif
             _PyStackRef *target_local = &GETLOCAL(next_oparg);
             assert(PyUnicode_CheckExact(left_o));
@@ -4692,7 +4692,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             right = _stack_item_1;
             left = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
             PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
             _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr;
@@ -4730,7 +4730,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             right = _stack_item_1;
             left = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
             PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
             assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5);
@@ -6736,7 +6736,7 @@
         case _GUARD_GLOBALS_VERSION_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            uint16_t version = (uint16_t)CURRENT_OPERAND0();
+            uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             if (!PyDict_CheckExact(dict)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -6759,7 +6759,7 @@
             CHECK_CURRENT_CACHED_VALUES(1);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
-            uint16_t version = (uint16_t)CURRENT_OPERAND0();
+            uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             if (!PyDict_CheckExact(dict)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -6784,7 +6784,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            uint16_t version = (uint16_t)CURRENT_OPERAND0();
+            uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             if (!PyDict_CheckExact(dict)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -6811,7 +6811,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
-            uint16_t version = (uint16_t)CURRENT_OPERAND0();
+            uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             if (!PyDict_CheckExact(dict)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -6837,8 +6837,8 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef res;
-            uint16_t version = (uint16_t)CURRENT_OPERAND0();
-            uint16_t index = (uint16_t)CURRENT_OPERAND1();
+            uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
+            uint16_t index = (uint16_t)CURRENT_OPERAND1_16();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             if (!PyDict_CheckExact(dict)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -6883,8 +6883,8 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef res;
-            uint16_t version = (uint16_t)CURRENT_OPERAND0();
-            uint16_t index = (uint16_t)CURRENT_OPERAND1();
+            uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
+            uint16_t index = (uint16_t)CURRENT_OPERAND1_16();
             PyDictObject *dict = (PyDictObject *)BUILTINS();
             if (!PyDict_CheckExact(dict)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -7936,7 +7936,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             assert(type_version != 0);
             if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@@ -7958,7 +7958,7 @@
             _PyStackRef owner;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             assert(type_version != 0);
             if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@@ -7979,7 +7979,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             assert(type_version != 0);
             if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@@ -8002,7 +8002,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             owner = _stack_item_2;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             assert(type_version != 0);
             if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@@ -8023,7 +8023,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             assert(type_version != 0);
             if (!LOCK_OBJECT(owner_o)) {
@@ -8054,7 +8054,7 @@
             _PyStackRef owner;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             assert(type_version != 0);
             if (!LOCK_OBJECT(owner_o)) {
@@ -8084,7 +8084,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             assert(type_version != 0);
             if (!LOCK_OBJECT(owner_o)) {
@@ -8116,7 +8116,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             owner = _stack_item_2;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             assert(type_version != 0);
             if (!LOCK_OBJECT(owner_o)) {
@@ -8235,7 +8235,7 @@
             _PyStackRef attr;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint16_t offset = (uint16_t)CURRENT_OPERAND0();
+            uint16_t offset = (uint16_t)CURRENT_OPERAND0_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
             PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
@@ -8280,8 +8280,8 @@
             _PyStackRef attr;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint32_t dict_version = (uint32_t)CURRENT_OPERAND0();
-            uint16_t index = (uint16_t)CURRENT_OPERAND1();
+            uint32_t dict_version = (uint32_t)CURRENT_OPERAND0_32();
+            uint16_t index = (uint16_t)CURRENT_OPERAND1_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
                 UOP_STAT_INC(uopcode, miss);
@@ -8342,7 +8342,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            uint16_t hint = (uint16_t)CURRENT_OPERAND0();
+            uint16_t hint = (uint16_t)CURRENT_OPERAND0_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
             PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
@@ -8428,7 +8428,7 @@
             _PyStackRef attr;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint16_t index = (uint16_t)CURRENT_OPERAND0();
+            uint16_t index = (uint16_t)CURRENT_OPERAND0_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             PyObject **addr = (PyObject **)((char *)owner_o + index);
             PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
@@ -8472,7 +8472,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             if (!PyType_Check(owner_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -8499,7 +8499,7 @@
             _PyStackRef owner;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             if (!PyType_Check(owner_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -8525,7 +8525,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             if (!PyType_Check(owner_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -8553,7 +8553,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             owner = _stack_item_2;
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             if (!PyType_Check(owner_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -8581,7 +8581,7 @@
             _PyStackRef attr;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
             attr = PyStackRef_FromPyObjectNew(descr);
@@ -8612,7 +8612,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            PyObject *fget = (PyObject *)CURRENT_OPERAND0();
+            PyObject *fget = (PyObject *)CURRENT_OPERAND0_64();
             assert((oparg & 1) == 0);
             assert(Py_IS_TYPE(fget, &PyFunction_Type));
             PyFunctionObject *f = (PyFunctionObject *)fget;
@@ -8763,7 +8763,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
             value = _stack_item_0;
-            uint16_t offset = (uint16_t)CURRENT_OPERAND0();
+            uint16_t offset = (uint16_t)CURRENT_OPERAND0_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             STAT_INC(STORE_ATTR, hit);
             assert(_PyObject_GetManagedDict(owner_o) == NULL);
@@ -8798,7 +8798,7 @@
             oparg = CURRENT_OPARG();
             owner = _stack_item_1;
             value = _stack_item_0;
-            uint16_t hint = (uint16_t)CURRENT_OPERAND0();
+            uint16_t hint = (uint16_t)CURRENT_OPERAND0_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
             PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
@@ -8874,7 +8874,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
             value = _stack_item_0;
-            uint16_t index = (uint16_t)CURRENT_OPERAND0();
+            uint16_t index = (uint16_t)CURRENT_OPERAND0_16();
             PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
             if (!LOCK_OBJECT(owner_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -11133,7 +11133,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef owner;
             owner = stack_pointer[-1];
-            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
             PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@@ -11156,7 +11156,7 @@
             _PyStackRef owner;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
             PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@@ -11178,7 +11178,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
-            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
             PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@@ -11202,7 +11202,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             owner = _stack_item_2;
-            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
             PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
             PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
             PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@@ -11227,7 +11227,7 @@
             _PyStackRef self;
             oparg = CURRENT_OPARG();
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11252,7 +11252,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11276,7 +11276,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             oparg = CURRENT_OPARG();
             owner = _stack_item_1;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11299,7 +11299,7 @@
             _PyStackRef self;
             oparg = CURRENT_OPARG();
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
             STAT_INC(LOAD_ATTR, hit);
@@ -11325,7 +11325,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
             STAT_INC(LOAD_ATTR, hit);
@@ -11350,7 +11350,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             oparg = CURRENT_OPARG();
             owner = _stack_item_1;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
             STAT_INC(LOAD_ATTR, hit);
@@ -11374,7 +11374,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert((oparg & 1) == 0);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11398,7 +11398,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert((oparg & 1) == 0);
             assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
             STAT_INC(LOAD_ATTR, hit);
@@ -11420,7 +11420,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef owner;
             owner = stack_pointer[-1];
-            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
+            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
             char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
             PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
             if (dict != NULL) {
@@ -11442,7 +11442,7 @@
             _PyStackRef owner;
             _PyStackRef _stack_item_0 = _tos_cache0;
             owner = _stack_item_0;
-            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
+            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
             char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
             PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
             if (dict != NULL) {
@@ -11463,7 +11463,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             owner = _stack_item_1;
-            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
+            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
             char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
             PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
             if (dict != NULL) {
@@ -11486,7 +11486,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             owner = _stack_item_2;
-            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
+            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
             char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
             PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
             if (dict != NULL) {
@@ -11510,7 +11510,7 @@
             _PyStackRef self;
             oparg = CURRENT_OPARG();
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11535,7 +11535,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             owner = _stack_item_0;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11559,7 +11559,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             oparg = CURRENT_OPARG();
             owner = _stack_item_1;
-            PyObject *descr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
@@ -11656,7 +11656,7 @@
             _PyStackRef callable;
             oparg = CURRENT_OPARG();
             callable = stack_pointer[-2 - oparg];
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             if (!PyFunction_Check(callable_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -11677,8 +11677,8 @@
         case _CHECK_FUNCTION_VERSION_INLINE_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
-            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
+            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
             assert(PyFunction_Check(callable_o));
             PyFunctionObject *func = (PyFunctionObject *)callable_o;
             if (func->func_version != func_version) {
@@ -11695,8 +11695,8 @@
             CHECK_CURRENT_CACHED_VALUES(1);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
-            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
+            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
             assert(PyFunction_Check(callable_o));
             PyFunctionObject *func = (PyFunctionObject *)callable_o;
             if (func->func_version != func_version) {
@@ -11715,8 +11715,8 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
-            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
+            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
             assert(PyFunction_Check(callable_o));
             PyFunctionObject *func = (PyFunctionObject *)callable_o;
             if (func->func_version != func_version) {
@@ -11737,8 +11737,8 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
-            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
+            PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
             assert(PyFunction_Check(callable_o));
             PyFunctionObject *func = (PyFunctionObject *)callable_o;
             if (func->func_version != func_version) {
@@ -11762,7 +11762,7 @@
             oparg = CURRENT_OPARG();
             null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             if (Py_TYPE(callable_o) != &PyMethod_Type) {
                 UOP_STAT_INC(uopcode, miss);
@@ -12941,7 +12941,7 @@
             oparg = CURRENT_OPARG();
             self_or_null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
-            uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             if (!PyStackRef_IsNull(self_or_null)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -14123,7 +14123,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             oparg = CURRENT_OPARG();
             callable = stack_pointer[-2 - oparg];
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             if (!PyFunction_Check(callable_o)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -14151,7 +14151,7 @@
             oparg = CURRENT_OPARG();
             null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
-            uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             if (Py_TYPE(callable_o) != &PyMethod_Type) {
                 UOP_STAT_INC(uopcode, miss);
@@ -15403,7 +15403,7 @@
         case _SET_IP_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
             frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
             SET_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
@@ -15414,7 +15414,7 @@
             CHECK_CURRENT_CACHED_VALUES(1);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
-            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
             frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
             _tos_cache0 = _stack_item_0;
             SET_CURRENT_CACHED_VALUES(1);
@@ -15427,7 +15427,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
             frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
             _tos_cache1 = _stack_item_1;
             _tos_cache0 = _stack_item_0;
@@ -15442,7 +15442,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
-            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
             frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
             _tos_cache2 = _stack_item_2;
             _tos_cache1 = _stack_item_1;
@@ -15455,7 +15455,7 @@
         case _CHECK_STACK_SPACE_OPERAND_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
+            uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
             assert(framesize <= INT_MAX);
             if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -15476,7 +15476,7 @@
             CHECK_CURRENT_CACHED_VALUES(1);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
-            uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
+            uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
             assert(framesize <= INT_MAX);
             if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -15499,7 +15499,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
+            uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
             assert(framesize <= INT_MAX);
             if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -15524,7 +15524,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
-            uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
+            uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
             assert(framesize <= INT_MAX);
             if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
                 UOP_STAT_INC(uopcode, miss);
@@ -15619,7 +15619,7 @@
         case _EXIT_TRACE_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             _PyExitData *exit = (_PyExitData *)exit_p;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@@ -15646,7 +15646,7 @@
             CHECK_CURRENT_CACHED_VALUES(1);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             _PyExitData *exit = (_PyExitData *)exit_p;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@@ -15681,7 +15681,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             _PyExitData *exit = (_PyExitData *)exit_p;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@@ -15719,7 +15719,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             _PyExitData *exit = (_PyExitData *)exit_p;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@@ -15756,7 +15756,7 @@
         case _DYNAMIC_EXIT_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             _PyExitData *exit = (_PyExitData *)exit_p;
             _Py_CODEUNIT *target = frame->instr_ptr;
@@ -15780,7 +15780,7 @@
             CHECK_CURRENT_CACHED_VALUES(1);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             _PyExitData *exit = (_PyExitData *)exit_p;
             _Py_CODEUNIT *target = frame->instr_ptr;
@@ -15812,7 +15812,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             _PyExitData *exit = (_PyExitData *)exit_p;
             _Py_CODEUNIT *target = frame->instr_ptr;
@@ -15847,7 +15847,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
-            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             _PyExitData *exit = (_PyExitData *)exit_p;
             _Py_CODEUNIT *target = frame->instr_ptr;
@@ -15946,7 +15946,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef value;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             value = PyStackRef_FromPyObjectNew(ptr);
             _tos_cache0 = value;
             SET_CURRENT_CACHED_VALUES(1);
@@ -15959,7 +15959,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef value;
             _PyStackRef _stack_item_0 = _tos_cache0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             value = PyStackRef_FromPyObjectNew(ptr);
             _tos_cache1 = value;
             _tos_cache0 = _stack_item_0;
@@ -15974,7 +15974,7 @@
             _PyStackRef value;
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             value = PyStackRef_FromPyObjectNew(ptr);
             _tos_cache2 = value;
             _tos_cache1 = _stack_item_1;
@@ -15991,7 +15991,7 @@
             _PyStackRef value;
             _PyStackRef _stack_item_0 = _tos_cache0;
             pop = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(pop);
             stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -16008,7 +16008,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef value;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             value = PyStackRef_FromPyObjectBorrow(ptr);
             _tos_cache0 = value;
             SET_CURRENT_CACHED_VALUES(1);
@@ -16021,7 +16021,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef value;
             _PyStackRef _stack_item_0 = _tos_cache0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             value = PyStackRef_FromPyObjectBorrow(ptr);
             _tos_cache1 = value;
             _tos_cache0 = _stack_item_0;
@@ -16036,7 +16036,7 @@
             _PyStackRef value;
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             value = PyStackRef_FromPyObjectBorrow(ptr);
             _tos_cache2 = value;
             _tos_cache1 = _stack_item_1;
@@ -16147,7 +16147,7 @@
             _PyStackRef value;
             _PyStackRef _stack_item_0 = _tos_cache0;
             pop = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(pop);
             stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -16170,7 +16170,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             pop2 = _stack_item_1;
             pop1 = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             stack_pointer[0] = pop1;
             stack_pointer += 1;
             ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@@ -16201,7 +16201,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             null = _stack_item_1;
             callable = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             (void)null;
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(callable);
@@ -16228,7 +16228,7 @@
             pop = _stack_item_2;
             null = _stack_item_1;
             callable = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             stack_pointer[0] = callable;
             stack_pointer[1] = null;
             stack_pointer += 2;
@@ -16266,7 +16266,7 @@
             pop1 = _stack_item_1;
             null = _stack_item_0;
             callable = stack_pointer[-1];
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             stack_pointer[0] = null;
             stack_pointer[1] = pop1;
             stack_pointer += 2;
@@ -16301,7 +16301,7 @@
             _PyStackRef value;
             _PyStackRef new;
             old = stack_pointer[-1];
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             new = old;
             value = PyStackRef_FromPyObjectNew(ptr);
             _tos_cache1 = new;
@@ -16321,7 +16321,7 @@
             _PyStackRef new;
             _PyStackRef _stack_item_0 = _tos_cache0;
             old = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             new = old;
             value = PyStackRef_FromPyObjectNew(ptr);
             _tos_cache1 = new;
@@ -16340,7 +16340,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             old = _stack_item_1;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             new = old;
             value = PyStackRef_FromPyObjectNew(ptr);
             _tos_cache2 = new;
@@ -16358,7 +16358,7 @@
             _PyStackRef value;
             _PyStackRef new;
             old = stack_pointer[-1];
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             new = old;
             value = PyStackRef_FromPyObjectBorrow(ptr);
             _tos_cache1 = new;
@@ -16378,7 +16378,7 @@
             _PyStackRef new;
             _PyStackRef _stack_item_0 = _tos_cache0;
             old = _stack_item_0;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             new = old;
             value = PyStackRef_FromPyObjectBorrow(ptr);
             _tos_cache1 = new;
@@ -16397,7 +16397,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             old = _stack_item_1;
-            PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
             new = old;
             value = PyStackRef_FromPyObjectBorrow(ptr);
             _tos_cache2 = new;
@@ -16411,7 +16411,7 @@
         case _START_EXECUTOR_r00: {
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
-            PyObject *executor = (PyObject *)CURRENT_OPERAND0();
+            PyObject *executor = (PyObject *)CURRENT_OPERAND0_64();
             #ifndef _Py_JIT
             assert(current_executor == (_PyExecutorObject*)executor);
             #endif
@@ -16646,7 +16646,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             oparg = CURRENT_OPARG();
-            uint32_t target = (uint32_t)CURRENT_OPERAND0();
+            uint32_t target = (uint32_t)CURRENT_OPERAND0_32();
             assert(oparg == 0);
             frame->instr_ptr = _PyFrame_GetBytecode(frame) + target;
             SET_CURRENT_CACHED_VALUES(0);
@@ -16972,7 +16972,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             #define OFFSET_OF__PUSH_FRAME ((0))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@@ -16993,7 +16993,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             #define OFFSET_OF__PUSH_FRAME ((0))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@@ -17016,7 +17016,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             #define OFFSET_OF__PUSH_FRAME ((0))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@@ -17041,7 +17041,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             #define OFFSET_OF__PUSH_FRAME ((0))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@@ -17064,7 +17064,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@@ -17085,7 +17085,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@@ -17108,7 +17108,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@@ -17133,7 +17133,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@@ -17156,7 +17156,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             #define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@@ -17177,7 +17177,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             #define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@@ -17200,7 +17200,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             #define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@@ -17225,7 +17225,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             #define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@@ -17248,7 +17248,7 @@
             CHECK_CURRENT_CACHED_VALUES(0);
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
@@ -17269,7 +17269,7 @@
             assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
             _PyStackRef _stack_item_0 = _tos_cache0;
             #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
@@ -17292,7 +17292,7 @@
             _PyStackRef _stack_item_0 = _tos_cache0;
             _PyStackRef _stack_item_1 = _tos_cache1;
             #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
@@ -17317,7 +17317,7 @@
             _PyStackRef _stack_item_1 = _tos_cache1;
             _PyStackRef _stack_item_2 = _tos_cache2;
             #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
-            PyObject *ip = (PyObject *)CURRENT_OPERAND0();
+            PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
             _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
             if (target != (_Py_CODEUNIT *)ip) {
                 frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 420cdcaed033397a57495d6544c66fd2ccaf0a36..ba80886f6dc0c7c380fd94ad1460005b703d607c 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -371,7 +371,7 @@
                 assert(next_instr->op.code == STORE_FAST);
                 next_oparg = next_instr->op.arg;
                 #else
-                next_oparg = (int)CURRENT_OPERAND0();
+                next_oparg = (int)CURRENT_OPERAND0_16();
                 #endif
                 _PyStackRef *target_local = &GETLOCAL(next_oparg);
                 assert(PyUnicode_CheckExact(left_o));

diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py
index 7e50a6ebafbfd332f779b7e69e93a01719143b02..9c86382cdc5610df9eb2561a1187ed3d262b1a2f 100644 (file)
--- a/Tools/cases_generator/tier2_generator.py
+++ b/Tools/cases_generator/tier2_generator.py
@@ -222,12 +222,13 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, t
         idx = 0
         for cache in uop.caches:
             if cache.name != "unused":
+                bits = cache.size*16
                 if cache.size == 4:
                     type = cast = "PyObject *"
                 else:
-                    type = f"uint{cache.size*16}_t "
-                    cast = f"uint{cache.size*16}_t"
-                emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n")
+                    type = f"uint{bits}_t "
+                    cast = f"uint{bits}_t"
+                emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}_{bits}();\n")
                 idx += 1
         reachable, storage = emitter.emit_tokens(uop, storage, None, False)
         if reachable:

diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py
index 297b9517f6a27ae06a0977c158c9559a1788115d..08d4e700652deb9f3a089cd1ec113f22038618f2 100644 (file)
--- a/Tools/jit/_optimizers.py
+++ b/Tools/jit/_optimizers.py
@@ -96,6 +96,8 @@ class InstructionKind(enum.Enum):
     LONG_BRANCH = enum.auto()
     SHORT_BRANCH = enum.auto()
     RETURN = enum.auto()
+    SMALL_CONST_1 = enum.auto()
+    SMALL_CONST_2 = enum.auto()
     OTHER = enum.auto()
 
 
@@ -172,6 +174,7 @@ class Optimizer:
     )
     # Override everything that follows in subclasses:
     _supports_external_relocations = True
+    supports_small_constants = False
     _branches: typing.ClassVar[dict[str, tuple[str | None, str | None]]] = {}
     # Short branches are instructions that can branch within a micro-op,
     # but might not have the reach to branch anywhere within a trace.
@@ -184,6 +187,9 @@ class Optimizer:
     _re_return: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH
     text: str = ""
     globals: set[str] = dataclasses.field(default_factory=set)
+    _re_small_const_1 = _RE_NEVER_MATCH
+    _re_small_const_2 = _RE_NEVER_MATCH
+    const_reloc = "<Not supported>"
 
     def __post_init__(self) -> None:
         # Split the code into a linked list of basic blocks. A basic block is an
@@ -253,6 +259,14 @@ class Optimizer:
         elif match := self._re_return.match(line):
             name = line
             kind = InstructionKind.RETURN
+        elif match := self._re_small_const_1.match(line):
+            target = match["value"]
+            name = match["instruction"]
+            kind = InstructionKind.SMALL_CONST_1
+        elif match := self._re_small_const_2.match(line):
+            target = match["value"]
+            name = match["instruction"]
+            kind = InstructionKind.SMALL_CONST_2
         else:
             name, *_ = line.split(" ")
             kind = InstructionKind.OTHER
@@ -385,7 +399,7 @@ class Optimizer:
                 block.fallthrough = True
                 block.instructions.pop()
             # Before:
-            #    br ? FOO:
+            #    branch  FOO:
             #    ...
             #    FOO:
             #    jump BAR
@@ -461,6 +475,70 @@ class Optimizer:
                     )
                     block.instructions.append(branch.update_target("0"))
 
+    def _make_temp_label(self, index: int) -> Instruction:
+        marker = f"jit_temp_{index}:"
+        return Instruction(InstructionKind.OTHER, "", marker, None)
+
+    def _fixup_constants(self) -> None:
+        if not self.supports_small_constants:
+            return
+        index = 0
+        for block in self._blocks():
+            fixed: list[Instruction] = []
+            small_const_index = -1
+            for inst in block.instructions:
+                if inst.kind == InstructionKind.SMALL_CONST_1:
+                    marker = f"jit_pending_{inst.target}{index}:"
+                    fixed.append(self._make_temp_label(index))
+                    index += 1
+                    small_const_index = len(fixed)
+                    fixed.append(inst)
+                elif inst.kind == InstructionKind.SMALL_CONST_2:
+                    if small_const_index < 0:
+                        fixed.append(inst)
+                        continue
+                    small_const_1 = fixed[small_const_index]
+                    if not self._small_consts_match(small_const_1, inst):
+                        small_const_index = -1
+                        fixed.append(inst)
+                        continue
+                    assert small_const_1.target is not None
+                    if small_const_1.target.endswith("16"):
+                        fixed[small_const_index] = self._make_temp_label(index)
+                        index += 1
+                    else:
+                        assert small_const_1.target.endswith("32")
+                        patch_kind, replacement = self._small_const_1(small_const_1)
+                        if replacement is not None:
+                            label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{small_const_1.target[:-3]}_JIT_RELOCATION_{index}:"
+                            index += 1
+                            fixed[small_const_index - 1] = Instruction(
+                                InstructionKind.OTHER, "", label, None
+                            )
+                            fixed[small_const_index] = replacement
+                    patch_kind, replacement = self._small_const_2(inst)
+                    if replacement is not None:
+                        assert inst.target is not None
+                        label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{inst.target[:-3]}_JIT_RELOCATION_{index}:"
+                        index += 1
+                        fixed.append(
+                            Instruction(InstructionKind.OTHER, "", label, None)
+                        )
+                        fixed.append(replacement)
+                    small_const_index = -1
+                else:
+                    fixed.append(inst)
+            block.instructions = fixed
+
+    def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]:
+        raise NotImplementedError()
+
+    def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]:
+        raise NotImplementedError()
+
+    def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool:
+        raise NotImplementedError()
+
     def run(self) -> None:
         """Run this optimizer."""
         self._insert_continue_label()
@@ -472,6 +550,7 @@ class Optimizer:
             self._remove_redundant_jumps()
             self._remove_unreachable()
         self._fixup_external_labels()
+        self._fixup_constants()
         self.path.write_text(self._body())
 
 
@@ -492,6 +571,54 @@ class OptimizerAArch64(Optimizer):  # pylint: disable = too-few-public-methods
     # https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/RET--Return-from-subroutine-
     _re_return = re.compile(r"\s*ret\b")
 
+    supports_small_constants = True
+    _re_small_const_1 = re.compile(
+        r"\s*(?P<instruction>adrp)\s+.*(?P<value>_JIT_OP(ARG|ERAND(0|1))_(16|32)).*"
+    )
+    _re_small_const_2 = re.compile(
+        r"\s*(?P<instruction>ldr)\s+.*(?P<value>_JIT_OP(ARG|ERAND(0|1))_(16|32)).*"
+    )
+    const_reloc = "CUSTOM_AARCH64_CONST"
+
+    def _get_reg(self, inst: Instruction) -> str:
+        _, rest = inst.text.split(inst.name)
+        reg, *_ = rest.split(",")
+        return reg.strip()
+
+    def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]:
+        assert inst.kind is InstructionKind.SMALL_CONST_1
+        assert inst.target is not None
+        if "16" in inst.target:
+            return "", None
+        pre, _ = inst.text.split(inst.name)
+        return "16a", Instruction(
+            InstructionKind.OTHER, "movz", f"{pre}movz {self._get_reg(inst)}, 0", None
+        )
+
+    def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]:
+        assert inst.kind is InstructionKind.SMALL_CONST_2
+        assert inst.target is not None
+        pre, _ = inst.text.split(inst.name)
+        if "16" in inst.target:
+            return "16a", Instruction(
+                InstructionKind.OTHER,
+                "movz",
+                f"{pre}movz {self._get_reg(inst)}, 0",
+                None,
+            )
+        else:
+            return "16b", Instruction(
+                InstructionKind.OTHER,
+                "movk",
+                f"{pre}movk {self._get_reg(inst)}, 0, lsl #16",
+                None,
+            )
+
+    def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool:
+        reg1 = self._get_reg(inst1)
+        reg2 = self._get_reg(inst2)
+        return reg1 == reg2
+
 
 class OptimizerX86(Optimizer):  # pylint: disable = too-few-public-methods
     """i686-pc-windows-msvc/x86_64-apple-darwin/x86_64-unknown-linux-gnu"""

diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py
index 4e86abe604972eb726d0fdc229136a7dd6826a0d..964e8bdbdc101024f2f5fe9a7851b01d0ecda2d5 100644 (file)
--- a/Tools/jit/_schema.py
+++ b/Tools/jit/_schema.py
@@ -9,6 +9,9 @@ HoleKind: typing.TypeAlias = typing.Literal[
     "ARM64_RELOC_PAGE21",
     "ARM64_RELOC_PAGEOFF12",
     "ARM64_RELOC_UNSIGNED",
+    "CUSTOM_AARCH64_BRANCH19",
+    "CUSTOM_AARCH64_CONST_16",
+    "CUSTOM_AARCH64_CONST_32",
     "IMAGE_REL_AMD64_REL32",
     "IMAGE_REL_ARM64_BRANCH19",
     "IMAGE_REL_ARM64_BRANCH26",

diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py
index cdffd953ef98389b36addab890cd04b88cec771c..2b78d8013af5dbc8e599c25ef037b7532c50f7b9 100644 (file)
--- a/Tools/jit/_stencils.py
+++ b/Tools/jit/_stencils.py
@@ -32,6 +32,12 @@ class HoleValue(enum.Enum):
     # The current uop's operand0 on 32-bit platforms (exposed as _JIT_OPERAND0_HI/LO):
     OPERAND0_HI = enum.auto()
     OPERAND0_LO = enum.auto()
+    # 16 and 32 bit versions of OPARG, OPERAND0 and OPERAND1
+    OPARG_16 = enum.auto()
+    OPERAND0_16 = enum.auto()
+    OPERAND1_16 = enum.auto()
+    OPERAND0_32 = enum.auto()
+    OPERAND1_32 = enum.auto()
     # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND1):
     OPERAND1 = enum.auto()
     # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND1_HI/LO):
@@ -59,6 +65,8 @@ _PATCH_FUNCS = {
     "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12",
     "ARM64_RELOC_UNSIGNED": "patch_64",
     "CUSTOM_AARCH64_BRANCH19": "patch_aarch64_19r",
+    "CUSTOM_AARCH64_CONST16a": "patch_aarch64_16a",
+    "CUSTOM_AARCH64_CONST16b": "patch_aarch64_16b",
     # x86_64-pc-windows-msvc:
     "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx",
     # aarch64-pc-windows-msvc:
@@ -95,6 +103,7 @@ _PATCH_FUNCS = {
     "X86_64_RELOC_SIGNED": "patch_32r",
     "X86_64_RELOC_UNSIGNED": "patch_64",
 }
+
 # Translate HoleValues to C expressions:
 _HOLE_EXPRS = {
     HoleValue.CODE: "(uintptr_t)code",
@@ -103,10 +112,15 @@ _HOLE_EXPRS = {
     HoleValue.GOT: "",
     # These should all have been turned into DATA values by process_relocations:
     HoleValue.OPARG: "instruction->oparg",
+    HoleValue.OPARG_16: "instruction->oparg",
     HoleValue.OPERAND0: "instruction->operand0",
+    HoleValue.OPERAND0_16: "instruction->operand0",
+    HoleValue.OPERAND0_32: "instruction->operand0",
     HoleValue.OPERAND0_HI: "(instruction->operand0 >> 32)",
     HoleValue.OPERAND0_LO: "(instruction->operand0 & UINT32_MAX)",
     HoleValue.OPERAND1: "instruction->operand1",
+    HoleValue.OPERAND1_16: "instruction->operand1",
+    HoleValue.OPERAND1_32: "instruction->operand1",
     HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)",
     HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)",
     HoleValue.TARGET: "instruction->target",
@@ -201,7 +215,10 @@ class Hole:
             if self.symbol:
                 if value:
                     value += " + "
-                value += f"(uintptr_t)&{self.symbol}"
+                if self.symbol.startswith("CONST"):
+                    value += f"instruction->{self.symbol[10:].lower()}"
+                else:
+                    value += f"(uintptr_t)&{self.symbol}"
             if _signed(self.addend) or not value:
                 if value:
                     value += " + "

diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py
index f92f3eac13bde5d644aef7ee3fd2ddcffba27aa3..5895e91c3c44ce77faf4165b7c27e1da97704017 100644 (file)
--- a/Tools/jit/_targets.py
+++ b/Tools/jit/_targets.py
@@ -138,6 +138,7 @@ class _Target(typing.Generic[_S, _R]):
             f"--target={self.triple}",
             "-DPy_BUILD_CORE_MODULE",
             "-D_DEBUG" if self.debug else "-DNDEBUG",
+            f"-DSUPPORTS_SMALL_CONSTS={1 if self.optimizer.supports_small_constants else 0}",
             f"-D_JIT_OPCODE={opname}",
             "-D_PyJIT_ACTIVE",
             "-D_Py_JIT",

diff --git a/Tools/jit/template.c b/Tools/jit/template.c
index 93067977791c428de30a8c41506a9f3a15de5167..064b401bc3aca4445850dac630fd50612c7b06ba 100644 (file)
--- a/Tools/jit/template.c
+++ b/Tools/jit/template.c
@@ -34,14 +34,38 @@
 
 #include "jit.h"
 
+
+#undef CURRENT_OPERAND0_64
+#define CURRENT_OPERAND0_64() (_operand0_64)
+
+#undef CURRENT_OPERAND1_64
+#define CURRENT_OPERAND1_64() (_operand1_64)
+
+
 #undef CURRENT_OPARG
+#undef CURRENT_OPERAND0_16
+#undef CURRENT_OPERAND0_32
+#undef CURRENT_OPERAND1_16
+#undef CURRENT_OPERAND1_32
+
+#if SUPPORTS_SMALL_CONSTS
+
+#define CURRENT_OPARG() (_oparg_16)
+#define CURRENT_OPERAND0_32() (_operand0_32)
+#define CURRENT_OPERAND0_16() (_operand0_16)
+#define CURRENT_OPERAND1_32() (_operand1_32)
+#define CURRENT_OPERAND1_16() (_operand1_16)
+
+#else
+
 #define CURRENT_OPARG() (_oparg)
+#define CURRENT_OPERAND0_32() (_operand0_64)
+#define CURRENT_OPERAND0_16() (_operand0_64)
+#define CURRENT_OPERAND1_32() (_operand1_64)
+#define CURRENT_OPERAND1_16() (_operand1_64)
 
-#undef CURRENT_OPERAND0
-#define CURRENT_OPERAND0() (_operand0)
+#endif
 
-#undef CURRENT_OPERAND1
-#define CURRENT_OPERAND1() (_operand1)
 
 #undef CURRENT_TARGET
 #define CURRENT_TARGET() (_target)
@@ -105,18 +129,26 @@ _JIT_ENTRY(
     int uopcode = _JIT_OPCODE;
     _Py_CODEUNIT *next_instr;
     // Other stuff we need handy:
-    PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
 #if SIZEOF_VOID_P == 8
-    PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0)
-    PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1)
+    PATCH_VALUE(uint64_t, _operand0_64, _JIT_OPERAND0)
+    PATCH_VALUE(uint64_t, _operand1_64, _JIT_OPERAND1)
 #else
     assert(SIZEOF_VOID_P == 4);
     PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI)
     PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO)
-    uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo;
+    uint64_t _operand0_64 = ((uint64_t)_operand0_hi << 32) | _operand0_lo;
     PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI)
     PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO)
-    uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo;
+    uint64_t _operand1_64 = ((uint64_t)_operand1_hi << 32) | _operand1_lo;
+#endif
+#if SUPPORTS_SMALL_CONSTS
+    PATCH_VALUE(uint32_t, _operand0_32, _JIT_OPERAND0_32)
+    PATCH_VALUE(uint32_t, _operand1_32, _JIT_OPERAND1_32)
+    PATCH_VALUE(uint16_t, _operand0_16, _JIT_OPERAND0_16)
+    PATCH_VALUE(uint16_t, _operand1_16, _JIT_OPERAND1_16)
+    PATCH_VALUE(uint16_t, _oparg_16, _JIT_OPARG_16)
+#else
+    PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
 #endif
     PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
     OPT_STAT_INC(uops_executed);