]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-112287: Speed up Tier 2 (uop) interpreter a little (#112286)
authorGuido van Rossum <guido@python.org>
Mon, 20 Nov 2023 19:25:32 +0000 (11:25 -0800)
committerGitHub <noreply@github.com>
Mon, 20 Nov 2023 19:25:32 +0000 (11:25 -0800)
This makes the Tier 2 interpreter a little faster.
I calculated by about 3%,
though I hesitate to claim an exact number.

This starts by doubling the trace size limit (to 512),
making it more likely that loops fit in a trace.

The rest of the approach is to only load
`oparg` and `operand` in cases that use them.
The code generator know when these are used.

For `oparg`, it will conditionally emit
```
oparg = CURRENT_OPARG();
```
at the top of the case block.
(The `oparg` variable may be referenced multiple times
by the instructions code block, so it must be in a variable.)

For `operand`, it will use `CURRENT_OPERAND()` directly
instead of referencing the `operand` variable,
which no longer exists.
(There is only one place where this will be used.)

Include/internal/pycore_uops.h
Misc/NEWS.d/next/Core and Builtins/2023-11-20-10-40-40.gh-issue-112287.15gWAK.rst [new file with mode: 0644]
Python/ceval.c
Python/ceval_macros.h
Python/executor_cases.c.h
Tools/cases_generator/generate_cases.py
Tools/cases_generator/instructions.py

index 8ab9aaf4108079739e90117f269abebbd9d41c57..e2b94894681f44d3a841738a579a1719924feb23 100644 (file)
@@ -10,7 +10,7 @@ extern "C" {
 
 #include "pycore_frame.h"         // _PyInterpreterFrame
 
-#define _Py_UOP_MAX_TRACE_LENGTH 256
+#define _Py_UOP_MAX_TRACE_LENGTH 512
 
 typedef struct {
     uint16_t opcode;
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-20-10-40-40.gh-issue-112287.15gWAK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-20-10-40-40.gh-issue-112287.15gWAK.rst
new file mode 100644 (file)
index 0000000..3f31a0f
--- /dev/null
@@ -0,0 +1,3 @@
+Slightly optimize the Tier 2 (uop) interpreter by only loading ``oparg`` and
+``operand`` when needed. Also double the trace size limit again, to 512 this
+time.
index b5e85930c81c9ea3bea0e53aac9da73c1454235d..b19b746834278aeb6f0bc52f9ad5dc81e304e968 100644 (file)
@@ -994,21 +994,18 @@ enter_tier_two:
 
     OPT_STAT_INC(traces_executed);
     _PyUOpInstruction *next_uop = current_executor->trace;
-    uint64_t operand;
 #ifdef Py_STATS
     uint64_t trace_uop_execution_counter = 0;
 #endif
 
     for (;;) {
         opcode = next_uop->opcode;
-        oparg = next_uop->oparg;
-        operand = next_uop->operand;
         DPRINTF(3,
                 "%4d: uop %s, oparg %d, operand %" PRIu64 ", target %d, stack_level %d\n",
                 (int)(next_uop - current_executor->trace),
                 _PyUopName(opcode),
-                oparg,
-                operand,
+                next_uop->oparg,
+                next_uop->operand,
                 next_uop->target,
                 (int)(stack_pointer - _PyFrame_Stackbase(frame)));
         next_uop++;
@@ -1025,8 +1022,9 @@ enter_tier_two:
             default:
 #ifdef Py_DEBUG
             {
-                fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 "\n",
-                        opcode, oparg, operand);
+                fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 " @ %d\n",
+                        opcode, next_uop[-1].oparg, next_uop[-1].operand,
+                        (int)(next_uop - current_executor->trace - 1));
                 Py_FatalError("Unknown uop");
             }
 #else
@@ -1055,7 +1053,7 @@ pop_1_error_tier_two:
     STACK_SHRINK(1);
 error_tier_two:
     DPRINTF(2, "Error: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n",
-            opcode, _PyUopName(opcode), oparg, operand, next_uop[-1].target,
+            opcode, _PyUopName(opcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target,
             (int)(next_uop - current_executor->trace - 1));
     OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
     frame->return_offset = 0;  // Don't leave this random
@@ -1068,7 +1066,7 @@ deoptimize:
     // On DEOPT_IF we just repeat the last instruction.
     // This presumes nothing was popped from the stack (nor pushed).
     DPRINTF(2, "DEOPT: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n",
-            opcode, _PyUopName(opcode), oparg, operand, next_uop[-1].target,
+            opcode, _PyUopName(opcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target,
             (int)(next_uop - current_executor->trace - 1));
     OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
     UOP_STAT_INC(opcode, miss);
index 546adbe5f438d1130b82c63c4b604af8a69f5167..b0cb7c8926338cca46aad340e611596782709a6d 100644 (file)
@@ -397,3 +397,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame);
 #define GOTO_TIER_TWO() goto enter_tier_two;
 
 #define GOTO_TIER_ONE() goto exit_trace;
+
+#define CURRENT_OPARG() (next_uop[-1].oparg)
+
+#define CURRENT_OPERAND() (next_uop[-1].operand)
index ae662b20e4403f2698522cdbff17c5a705f93c2b..547be6f13237dd0b118b80be0ee8dfe03b7a01eb 100644 (file)
@@ -25,6 +25,7 @@
         }
 
         case LOAD_FAST_CHECK: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             value = GETLOCAL(oparg);
             if (value == NULL) goto unbound_local_error_tier_two;
@@ -35,6 +36,7 @@
         }
 
         case LOAD_FAST: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             value = GETLOCAL(oparg);
             assert(value != NULL);
@@ -45,6 +47,7 @@
         }
 
         case LOAD_FAST_AND_CLEAR: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             value = GETLOCAL(oparg);
             // do not use SETLOCAL here, it decrefs the old value
@@ -55,6 +58,7 @@
         }
 
         case LOAD_CONST: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             value = GETITEM(FRAME_CO_CONSTS, oparg);
             Py_INCREF(value);
@@ -64,6 +68,7 @@
         }
 
         case STORE_FAST: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             value = stack_pointer[-1];
             SETLOCAL(oparg, value);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
-            uint32_t version = (uint32_t)operand;
+            uint32_t version = (uint32_t)CURRENT_OPERAND();
             // This one is a bit weird, because we expect *some* failures:
             assert(version);
             DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
         }
 
         case LIST_APPEND: {
+            oparg = CURRENT_OPARG();
             PyObject *v;
             PyObject *list;
             v = stack_pointer[-1];
         }
 
         case SET_ADD: {
+            oparg = CURRENT_OPARG();
             PyObject *v;
             PyObject *set;
             v = stack_pointer[-1];
         }
 
         case CALL_INTRINSIC_1: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
         }
 
         case CALL_INTRINSIC_2: {
+            oparg = CURRENT_OPARG();
             PyObject *value1;
             PyObject *value2;
             PyObject *res;
         }
 
         case GET_AWAITABLE: {
+            oparg = CURRENT_OPARG();
             PyObject *iterable;
             PyObject *iter;
             iterable = stack_pointer[-1];
         }
 
         case STORE_NAME: {
+            oparg = CURRENT_OPARG();
             PyObject *v;
             v = stack_pointer[-1];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
         }
 
         case DELETE_NAME: {
+            oparg = CURRENT_OPARG();
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             PyObject *ns = LOCALS();
             int err;
         }
 
         case _UNPACK_SEQUENCE: {
+            oparg = CURRENT_OPARG();
             PyObject *seq;
             seq = stack_pointer[-1];
             PyObject **top = stack_pointer + oparg - 1;
         }
 
         case UNPACK_SEQUENCE_TWO_TUPLE: {
+            oparg = CURRENT_OPARG();
             PyObject *seq;
             PyObject **values;
             seq = stack_pointer[-1];
         }
 
         case UNPACK_SEQUENCE_TUPLE: {
+            oparg = CURRENT_OPARG();
             PyObject *seq;
             PyObject **values;
             seq = stack_pointer[-1];
         }
 
         case UNPACK_SEQUENCE_LIST: {
+            oparg = CURRENT_OPARG();
             PyObject *seq;
             PyObject **values;
             seq = stack_pointer[-1];
         }
 
         case UNPACK_EX: {
+            oparg = CURRENT_OPARG();
             PyObject *seq;
             seq = stack_pointer[-1];
             int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
         }
 
         case _STORE_ATTR: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *v;
             owner = stack_pointer[-1];
         }
 
         case DELETE_ATTR: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             owner = stack_pointer[-1];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
         }
 
         case STORE_GLOBAL: {
+            oparg = CURRENT_OPARG();
             PyObject *v;
             v = stack_pointer[-1];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
         }
 
         case DELETE_GLOBAL: {
+            oparg = CURRENT_OPARG();
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err;
             err = PyDict_DelItem(GLOBALS(), name);
         }
 
         case LOAD_FROM_DICT_OR_GLOBALS: {
+            oparg = CURRENT_OPARG();
             PyObject *mod_or_class_dict;
             PyObject *v;
             mod_or_class_dict = stack_pointer[-1];
         }
 
         case LOAD_NAME: {
+            oparg = CURRENT_OPARG();
             PyObject *v;
             PyObject *mod_or_class_dict = LOCALS();
             if (mod_or_class_dict == NULL) {
         }
 
         case _LOAD_GLOBAL: {
+            oparg = CURRENT_OPARG();
             PyObject *res;
             PyObject *null = NULL;
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
         }
 
         case _GUARD_GLOBALS_VERSION: {
-            uint16_t version = (uint16_t)operand;
+            uint16_t version = (uint16_t)CURRENT_OPERAND();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION);
             DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION);
         }
 
         case _GUARD_BUILTINS_VERSION: {
-            uint16_t version = (uint16_t)operand;
+            uint16_t version = (uint16_t)CURRENT_OPERAND();
             PyDictObject *dict = (PyDictObject *)BUILTINS();
             DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION);
             DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION);
         }
 
         case _LOAD_GLOBAL_MODULE: {
+            oparg = CURRENT_OPARG();
             PyObject *res;
             PyObject *null = NULL;
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
             res = entries[index].me_value;
         }
 
         case _LOAD_GLOBAL_BUILTINS: {
+            oparg = CURRENT_OPARG();
             PyObject *res;
             PyObject *null = NULL;
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             PyDictObject *bdict = (PyDictObject *)BUILTINS();
             PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
             res = entries[index].me_value;
         }
 
         case DELETE_FAST: {
+            oparg = CURRENT_OPARG();
             PyObject *v = GETLOCAL(oparg);
             if (v == NULL) goto unbound_local_error_tier_two;
             SETLOCAL(oparg, NULL);
         }
 
         case MAKE_CELL: {
+            oparg = CURRENT_OPARG();
             // "initial" is probably NULL but not if it's an arg (or set
             // via PyFrame_LocalsToFast() before MAKE_CELL has run).
             PyObject *initial = GETLOCAL(oparg);
         }
 
         case DELETE_DEREF: {
+            oparg = CURRENT_OPARG();
             PyObject *cell = GETLOCAL(oparg);
             PyObject *oldobj = PyCell_GET(cell);
             // Can't use ERROR_IF here.
         }
 
         case LOAD_FROM_DICT_OR_DEREF: {
+            oparg = CURRENT_OPARG();
             PyObject *class_dict;
             PyObject *value;
             class_dict = stack_pointer[-1];
         }
 
         case LOAD_DEREF: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             PyObject *cell = GETLOCAL(oparg);
             value = PyCell_GET(cell);
         }
 
         case STORE_DEREF: {
+            oparg = CURRENT_OPARG();
             PyObject *v;
             v = stack_pointer[-1];
             PyObject *cell = GETLOCAL(oparg);
         }
 
         case COPY_FREE_VARS: {
+            oparg = CURRENT_OPARG();
             /* Copy closure variables to free variables */
             PyCodeObject *co = _PyFrame_GetCode(frame);
             assert(PyFunction_Check(frame->f_funcobj));
         }
 
         case BUILD_STRING: {
+            oparg = CURRENT_OPARG();
             PyObject **pieces;
             PyObject *str;
             pieces = stack_pointer - oparg;
         }
 
         case BUILD_TUPLE: {
+            oparg = CURRENT_OPARG();
             PyObject **values;
             PyObject *tup;
             values = stack_pointer - oparg;
         }
 
         case BUILD_LIST: {
+            oparg = CURRENT_OPARG();
             PyObject **values;
             PyObject *list;
             values = stack_pointer - oparg;
         }
 
         case LIST_EXTEND: {
+            oparg = CURRENT_OPARG();
             PyObject *iterable;
             PyObject *list;
             iterable = stack_pointer[-1];
         }
 
         case SET_UPDATE: {
+            oparg = CURRENT_OPARG();
             PyObject *iterable;
             PyObject *set;
             iterable = stack_pointer[-1];
         }
 
         case BUILD_SET: {
+            oparg = CURRENT_OPARG();
             PyObject **values;
             PyObject *set;
             values = stack_pointer - oparg;
         }
 
         case BUILD_MAP: {
+            oparg = CURRENT_OPARG();
             PyObject **values;
             PyObject *map;
             values = stack_pointer - oparg*2;
         }
 
         case BUILD_CONST_KEY_MAP: {
+            oparg = CURRENT_OPARG();
             PyObject *keys;
             PyObject **values;
             PyObject *map;
         }
 
         case DICT_UPDATE: {
+            oparg = CURRENT_OPARG();
             PyObject *update;
             PyObject *dict;
             update = stack_pointer[-1];
         }
 
         case DICT_MERGE: {
+            oparg = CURRENT_OPARG();
             PyObject *update;
             PyObject *dict;
             PyObject *callable;
         }
 
         case MAP_ADD: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             PyObject *key;
             PyObject *dict;
         }
 
         case LOAD_SUPER_ATTR_ATTR: {
+            oparg = CURRENT_OPARG();
             PyObject *self;
             PyObject *class;
             PyObject *global_super;
         }
 
         case LOAD_SUPER_ATTR_METHOD: {
+            oparg = CURRENT_OPARG();
             PyObject *self;
             PyObject *class;
             PyObject *global_super;
         }
 
         case _LOAD_ATTR: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *self_or_null = NULL;
         case _GUARD_TYPE_VERSION: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = (uint32_t)operand;
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND();
             PyTypeObject *tp = Py_TYPE(owner);
             assert(type_version != 0);
             DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION);
         }
 
         case _LOAD_ATTR_INSTANCE_VALUE: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             owner = stack_pointer[-1];
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
             attr = _PyDictOrValues_GetValues(dorv)->values[index];
             DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE);
         case _CHECK_ATTR_MODULE: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = (uint32_t)operand;
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND();
             DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE);
             PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
             assert(dict != NULL);
         }
 
         case _LOAD_ATTR_MODULE: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             owner = stack_pointer[-1];
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
             assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
             assert(index < dict->ma_keys->dk_nentries);
         }
 
         case _LOAD_ATTR_WITH_HINT: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             owner = stack_pointer[-1];
-            uint16_t hint = (uint16_t)operand;
+            uint16_t hint = (uint16_t)CURRENT_OPERAND();
             PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
             PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
             DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT);
         }
 
         case _LOAD_ATTR_SLOT: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             owner = stack_pointer[-1];
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             char *addr = (char *)owner + index;
             attr = *(PyObject **)addr;
             DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT);
         case _CHECK_ATTR_CLASS: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = (uint32_t)operand;
+            uint32_t type_version = (uint32_t)CURRENT_OPERAND();
             DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS);
             assert(type_version != 0);
             DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS);
         }
 
         case _LOAD_ATTR_CLASS: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)operand;
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
             attr = Py_NewRef(descr);
             PyObject *value;
             owner = stack_pointer[-1];
             value = stack_pointer[-2];
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
             STAT_INC(STORE_ATTR, hit);
             PyDictValues *values = _PyDictOrValues_GetValues(dorv);
             PyObject *value;
             owner = stack_pointer[-1];
             value = stack_pointer[-2];
-            uint16_t index = (uint16_t)operand;
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
             char *addr = (char *)owner + index;
             STAT_INC(STORE_ATTR, hit);
             PyObject *old_value = *(PyObject **)addr;
         }
 
         case _COMPARE_OP: {
+            oparg = CURRENT_OPARG();
             PyObject *right;
             PyObject *left;
             PyObject *res;
         }
 
         case COMPARE_OP_FLOAT: {
+            oparg = CURRENT_OPARG();
             PyObject *right;
             PyObject *left;
             PyObject *res;
         }
 
         case COMPARE_OP_INT: {
+            oparg = CURRENT_OPARG();
             PyObject *right;
             PyObject *left;
             PyObject *res;
         }
 
         case COMPARE_OP_STR: {
+            oparg = CURRENT_OPARG();
             PyObject *right;
             PyObject *left;
             PyObject *res;
         }
 
         case IS_OP: {
+            oparg = CURRENT_OPARG();
             PyObject *right;
             PyObject *left;
             PyObject *b;
         }
 
         case CONTAINS_OP: {
+            oparg = CURRENT_OPARG();
             PyObject *right;
             PyObject *left;
             PyObject *b;
         }
 
         case MATCH_CLASS: {
+            oparg = CURRENT_OPARG();
             PyObject *names;
             PyObject *type;
             PyObject *subject;
         case _GUARD_KEYS_VERSION: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            uint32_t keys_version = (uint32_t)operand;
+            uint32_t keys_version = (uint32_t)CURRENT_OPERAND();
             PyTypeObject *owner_cls = Py_TYPE(owner);
             PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
             DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION);
         }
 
         case _LOAD_ATTR_METHOD_WITH_VALUES: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *self;
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)operand;
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
             assert(oparg & 1);
             /* Cached method object */
             STAT_INC(LOAD_ATTR, hit);
         }
 
         case _LOAD_ATTR_METHOD_NO_DICT: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *self;
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)operand;
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
             assert(oparg & 1);
             assert(Py_TYPE(owner)->tp_dictoffset == 0);
             STAT_INC(LOAD_ATTR, hit);
         }
 
         case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)operand;
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
             assert((oparg & 1) == 0);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
         }
 
         case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)operand;
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
             assert((oparg & 1) == 0);
             assert(Py_TYPE(owner)->tp_dictoffset == 0);
             STAT_INC(LOAD_ATTR, hit);
         }
 
         case _LOAD_ATTR_METHOD_LAZY_DICT: {
+            oparg = CURRENT_OPARG();
             PyObject *owner;
             PyObject *attr;
             PyObject *self;
             owner = stack_pointer[-1];
-            PyObject *descr = (PyObject *)operand;
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
             assert(oparg & 1);
             STAT_INC(LOAD_ATTR, hit);
             assert(descr != NULL);
         }
 
         case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: {
+            oparg = CURRENT_OPARG();
             PyObject *null;
             PyObject *callable;
             null = stack_pointer[-1 - oparg];
         }
 
         case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: {
+            oparg = CURRENT_OPARG();
             PyObject *callable;
             PyObject *func;
             PyObject *self;
         }
 
         case _CHECK_FUNCTION_EXACT_ARGS: {
+            oparg = CURRENT_OPARG();
             PyObject *self_or_null;
             PyObject *callable;
             self_or_null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
-            uint32_t func_version = (uint32_t)operand;
+            uint32_t func_version = (uint32_t)CURRENT_OPERAND();
             DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS);
             PyFunctionObject *func = (PyFunctionObject *)callable;
             DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS);
         }
 
         case _CHECK_STACK_SPACE: {
+            oparg = CURRENT_OPARG();
             PyObject *callable;
             callable = stack_pointer[-2 - oparg];
             PyFunctionObject *func = (PyFunctionObject *)callable;
         }
 
         case _INIT_CALL_PY_EXACT_ARGS: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_TYPE_1: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *null;
             PyObject *callable;
         }
 
         case CALL_STR_1: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *null;
             PyObject *callable;
         }
 
         case CALL_TUPLE_1: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *null;
             PyObject *callable;
         }
 
         case CALL_BUILTIN_CLASS: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_BUILTIN_O: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_BUILTIN_FAST: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_BUILTIN_FAST_WITH_KEYWORDS: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_LEN: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_ISINSTANCE: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_METHOD_DESCRIPTOR_O: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_METHOD_DESCRIPTOR_NOARGS: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case CALL_METHOD_DESCRIPTOR_FAST: {
+            oparg = CURRENT_OPARG();
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
         }
 
         case SET_FUNCTION_ATTRIBUTE: {
+            oparg = CURRENT_OPARG();
             PyObject *func;
             PyObject *attr;
             func = stack_pointer[-1];
         }
 
         case BUILD_SLICE: {
+            oparg = CURRENT_OPARG();
             PyObject *step = NULL;
             PyObject *stop;
             PyObject *start;
         }
 
         case CONVERT_VALUE: {
+            oparg = CURRENT_OPARG();
             PyObject *value;
             PyObject *result;
             value = stack_pointer[-1];
         }
 
         case COPY: {
+            oparg = CURRENT_OPARG();
             PyObject *bottom;
             PyObject *top;
             bottom = stack_pointer[-1 - (oparg-1)];
         }
 
         case _BINARY_OP: {
+            oparg = CURRENT_OPARG();
             PyObject *rhs;
             PyObject *lhs;
             PyObject *res;
         }
 
         case SWAP: {
+            oparg = CURRENT_OPARG();
             PyObject *top;
             PyObject *bottom;
             top = stack_pointer[-1];
         }
 
         case _SET_IP: {
+            oparg = CURRENT_OPARG();
             TIER_TWO_ONLY
             // TODO: Put the code pointer in `operand` to avoid indirection via `frame`
             frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg;
         }
 
         case _SAVE_RETURN_OFFSET: {
+            oparg = CURRENT_OPARG();
             #if TIER_ONE
             frame->return_offset = (uint16_t)(next_instr - this_instr);
             #endif
         }
 
         case _INSERT: {
+            oparg = CURRENT_OPARG();
             PyObject *top;
             top = stack_pointer[-1];
             // Inserts TOS at position specified by oparg;
index 149558e16403648fc1b0a2c3a0e84480268eaae3..851bd2f53879e57d6c18d389ac3ccba612db6646 100644 (file)
@@ -810,6 +810,8 @@ class Generator(Analyzer):
                     n_uops += 1
                     self.out.emit("")
                     with self.out.block(f"case {instr.name}:"):
+                        if instr.instr_flags.HAS_ARG_FLAG:
+                            self.out.emit("oparg = CURRENT_OPARG();")
                         stacking.write_single_instr(instr, self.out, tier=TIER_TWO)
                         if instr.check_eval_breaker:
                             self.out.emit("CHECK_EVAL_BREAKER();")
index 9039ac5c6f127ee4f845b3330b2926680ac12063..149a08810e4ae55742334a4ad360cb2242f97443 100644 (file)
@@ -166,7 +166,7 @@ class Instruction:
                     f"{func}(&this_instr[{active.offset + 1}].cache);"
                 )
             else:
-                out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;")
+                out.emit(f"{typ}{ceffect.name} = ({typ.strip()})CURRENT_OPERAND();")
 
         # Write the body, substituting a goto for ERROR_IF() and other stuff
         assert dedent <= 0