GH-111485: Increment `next_instr` consistently at the start of the instruction. ...

diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
index e2ed9bad7833a54d6ad2c128751ac1dea79afbc1..27948749d58eb778d6ffa617fbca96c84549b636 100644 (file)
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -1356,8 +1356,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
     [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG },
     [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
-    [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
-    [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
+    [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
+    [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
     [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG },
     [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
@@ -1434,7 +1434,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
-    [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG },
+    [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
     [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_ZERO_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
@@ -1495,7 +1495,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG },
-    [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
+    [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
     [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG },
     [_IS_ITER_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, 0 },
@@ -1533,7 +1533,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
     [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
     [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
-    [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
+    [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG },
     [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
@@ -2047,6 +2047,7 @@ extern const uint8_t _PyOpcode_Caches[256];
 #ifdef NEED_OPCODE_METADATA
 const uint8_t _PyOpcode_Caches[256] = {
     [TO_BOOL] = 3,
+    [BINARY_OP_INPLACE_ADD_UNICODE] = 1,
     [BINARY_SUBSCR] = 1,
     [STORE_SUBSCR] = 1,
     [SEND] = 1,

diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py
index 790e6b1b1b91e6b3cc6773a3fedc6d0c24ae7ea3..475d7492040b7bf111dadaac4b4fdaf0a65cb51b 100644 (file)
--- a/Lib/test/test_generated_cases.py
+++ b/Lib/test/test_generated_cases.py
@@ -138,6 +138,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             spam();
             DISPATCH();
         }
@@ -152,6 +155,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *value;
             value = stack_pointer[-1];
             spam();
@@ -169,6 +175,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *res;
             spam();
             STACK_GROW(1);
@@ -186,6 +195,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -204,6 +216,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -225,6 +240,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *right;
             PyObject *left;
             PyObject *result;
@@ -249,6 +267,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP1) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP1);
             PREDICTED(OP1);
             static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size");
             PyObject *arg;
@@ -259,6 +280,9 @@ class TestGeneratedCases(unittest.TestCase):
         }
 
         TARGET(OP3) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP3);
             PyObject *arg;
             PyObject *res;
             arg = stack_pointer[-1];
@@ -278,6 +302,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             if (cond) goto label;
             DISPATCH();
         }
@@ -292,6 +319,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             if (cond) goto label;
             DISPATCH();
         }
@@ -306,6 +336,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -326,12 +359,14 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(OP);
             PyObject *value;
             value = stack_pointer[-1];
-            uint16_t counter = read_u16(&next_instr[0].cache);
-            uint32_t extra = read_u32(&next_instr[1].cache);
+            uint16_t counter = read_u16(&this_instr[1].cache);
+            uint32_t extra = read_u32(&this_instr[2].cache);
             STACK_SHRINK(1);
-            next_instr += 3;
             DISPATCH();
         }
     """
@@ -345,6 +380,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             goto somewhere;
         }
     """
@@ -366,18 +404,24 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP1) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(OP1);
             PyObject *right;
             PyObject *left;
             right = stack_pointer[-1];
             left = stack_pointer[-2];
-            uint16_t counter = read_u16(&next_instr[0].cache);
+            uint16_t counter = read_u16(&this_instr[1].cache);
             op1(left, right);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 6;
+            INSTRUCTION_STATS(OP);
             PREDICTED(OP);
+            _Py_CODEUNIT *this_instr = next_instr - 6;
             static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size");
             PyObject *right;
             PyObject *left;
@@ -387,22 +431,24 @@ class TestGeneratedCases(unittest.TestCase):
             right = stack_pointer[-1];
             left = stack_pointer[-2];
             {
-                uint16_t counter = read_u16(&next_instr[0].cache);
+                uint16_t counter = read_u16(&this_instr[1].cache);
                 op1(left, right);
             }
             // OP2
             arg2 = stack_pointer[-3];
             {
-                uint32_t extra = read_u32(&next_instr[3].cache);
+                uint32_t extra = read_u32(&this_instr[4].cache);
                 res = op2(arg2, left, right);
             }
             STACK_SHRINK(2);
             stack_pointer[-1] = res;
-            next_instr += 5;
             DISPATCH();
         }
 
         TARGET(OP3) {
+            frame->instr_ptr = next_instr;
+            next_instr += 6;
+            INSTRUCTION_STATS(OP3);
             PyObject *right;
             PyObject *left;
             PyObject *arg2;
@@ -413,7 +459,6 @@ class TestGeneratedCases(unittest.TestCase):
             res = op3(arg2, left, right);
             STACK_SHRINK(2);
             stack_pointer[-1] = res;
-            next_instr += 5;
             DISPATCH();
         }
     """
@@ -427,6 +472,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *above;
             PyObject **values;
             PyObject *below;
@@ -449,6 +497,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *below;
             PyObject **values;
             PyObject *above;
@@ -470,6 +521,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject **values;
             PyObject *above;
             values = stack_pointer - oparg;
@@ -489,6 +543,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject **values;
             PyObject *extra;
             values = stack_pointer - oparg;
@@ -509,6 +566,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             PyObject *cc;
             PyObject *input = NULL;
             PyObject *aa;
@@ -541,6 +601,9 @@ class TestGeneratedCases(unittest.TestCase):
     """
         output = """
         TARGET(M) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(M);
             PyObject *right;
             PyObject *middle;
             PyObject *left;
@@ -580,6 +643,9 @@ class TestGeneratedCases(unittest.TestCase):
         """
         output = """
         TARGET(M) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(M);
             PyObject *val1;
             PyObject *val2;
             // A
@@ -609,6 +675,9 @@ class TestGeneratedCases(unittest.TestCase):
         """
         output = """
         TARGET(OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(OP);
             ham();
             DISPATCH();
         }
@@ -627,6 +696,9 @@ class TestGeneratedCases(unittest.TestCase):
         """
         output = """
         TARGET(M) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(M);
             ham();
             DISPATCH();
         }

diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py
index 3b8ecb765c23f0cf07ef5c6a32c509a74420c206..d12cd9637dd96e208ab37f7195f94c972cab0aaa 100644 (file)
--- a/Lib/test/test_monitoring.py
+++ b/Lib/test/test_monitoring.py
@@ -1378,15 +1378,16 @@ class TestBranchAndJumpEvents(CheckEvents):
                     x = 4
                 else:
                     x = 6
+            7
 
         self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [
             ('branch', 'func', 2, 2),
-            ('branch', 'func', 3, 4),
+            ('branch', 'func', 3, 6),
             ('jump', 'func', 6, 2),
             ('branch', 'func', 2, 2),
-            ('branch', 'func', 3, 3),
+            ('branch', 'func', 3, 4),
             ('jump', 'func', 4, 2),
-            ('branch', 'func', 2, 2)])
+            ('branch', 'func', 2, 7)])
 
         self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
             ('line', 'get_events', 10),
@@ -1394,17 +1395,18 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('line', 'func', 2),
             ('branch', 'func', 2, 2),
             ('line', 'func', 3),
-            ('branch', 'func', 3, 4),
+            ('branch', 'func', 3, 6),
             ('line', 'func', 6),
             ('jump', 'func', 6, 2),
             ('line', 'func', 2),
             ('branch', 'func', 2, 2),
             ('line', 'func', 3),
-            ('branch', 'func', 3, 3),
+            ('branch', 'func', 3, 4),
             ('line', 'func', 4),
             ('jump', 'func', 4, 2),
             ('line', 'func', 2),
-            ('branch', 'func', 2, 2),
+            ('branch', 'func', 2, 7),
+            ('line', 'func', 7),
             ('line', 'get_events', 11)])
 
     def test_except_star(self):
@@ -1434,7 +1436,7 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('line', 'meth', 1),
             ('jump', 'func', 5, 5),
             ('jump', 'func', 5, '[offset=114]'),
-            ('branch', 'func', '[offset=120]', '[offset=122]'),
+            ('branch', 'func', '[offset=120]', '[offset=124]'),
             ('line', 'get_events', 11)])
 
         self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [
@@ -1450,7 +1452,7 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('return', None),
             ('jump', 'func', 5, 5),
             ('jump', 'func', 5, '[offset=114]'),
-            ('branch', 'func', '[offset=120]', '[offset=122]'),
+            ('branch', 'func', '[offset=120]', '[offset=124]'),
             ('return', None),
             ('line', 'get_events', 11)])
 

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 5b1d70b303060d2c86a5d6289b1f3770328af4ab..d33ee3265d254d254aee4288e3651fd8bd14ec02 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -80,6 +80,7 @@ dummy_func(
     pop_1_error:
     // Dummy locals.
     PyObject *dummy;
+    _Py_CODEUNIT *this_instr;
     PyObject *attr;
     PyObject *attrs;
     PyObject *bottom;
@@ -144,13 +145,13 @@ dummy_func(
             if (code_version != global_version) {
                 int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
                 ERROR_IF(err, error);
-                next_instr--;
+                next_instr = this_instr;
             }
             else {
                 if (oparg < RESUME_AFTER_YIELD_FROM) {
                     CHECK_EVAL_BREAKER();
                 }
-                next_instr[-1].op.code = RESUME_CHECK;
+                this_instr->op.code = RESUME_CHECK;
             }
         }
 
@@ -172,7 +173,7 @@ dummy_func(
                 if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
                     goto error;
                 }
-                next_instr--;
+                next_instr = this_instr;
             }
             else {
                 if (oparg < 2) {
@@ -180,12 +181,12 @@ dummy_func(
                 }
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 int err = _Py_call_instrumentation(
-                        tstate, oparg > 0, frame, next_instr-1);
+                        tstate, oparg > 0, frame, this_instr);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
                 ERROR_IF(err, error);
-                if (frame->instr_ptr != next_instr-1) {
+                if (frame->instr_ptr != this_instr) {
                     /* Instrumentation has jumped */
-                    next_instr = frame->instr_ptr;
+                    next_instr = this_instr;
                     DISPATCH();
                 }
             }
@@ -261,11 +262,12 @@ dummy_func(
         macro(END_FOR) = POP_TOP + POP_TOP;
 
         inst(INSTRUMENTED_END_FOR, (receiver, value --)) {
+            TIER_ONE_ONLY
             /* Need to create a fake StopIteration error here,
              * to conform to PEP 380 */
             if (PyGen_Check(receiver)) {
                 PyErr_SetObject(PyExc_StopIteration, value);
-                if (monitor_stop_iteration(tstate, frame, next_instr-1)) {
+                if (monitor_stop_iteration(tstate, frame, this_instr)) {
                     goto error;
                 }
                 PyErr_SetRaisedException(NULL);
@@ -278,9 +280,10 @@ dummy_func(
         }
 
         inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) {
+            TIER_ONE_ONLY
             if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
                 PyErr_SetObject(PyExc_StopIteration, value);
-                if (monitor_stop_iteration(tstate, frame, next_instr-1)) {
+                if (monitor_stop_iteration(tstate, frame, this_instr)) {
                     goto error;
                 }
                 PyErr_SetRaisedException(NULL);
@@ -310,14 +313,13 @@ dummy_func(
 
         inst(TO_BOOL, (unused/1, unused/2, value -- res)) {
             #if ENABLE_SPECIALIZATION
-            _PyToBoolCache *cache = (_PyToBoolCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_ToBool(value, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(TO_BOOL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             int err = PyObject_IsTrue(value);
             DECREF_INPUTS();
@@ -491,10 +493,9 @@ dummy_func(
         // So the inputs are the same as for all BINARY_OP
         // specializations, but there is no output.
         // At the end we just skip over the STORE_FAST.
-        op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) {
-            _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
-            assert(true_next.op.code == STORE_FAST);
-            PyObject **target_local = &GETLOCAL(true_next.op.arg);
+        op(_BINARY_OP_INPLACE_ADD_UNICODE, (unused/1, left, right --)) {
+            assert(next_instr->op.code == STORE_FAST);
+            PyObject **target_local = &GETLOCAL(next_instr->op.arg);
             DEOPT_IF(*target_local != left);
             STAT_INC(BINARY_OP, hit);
             /* Handle `left = left + right` or `left += right` for str.
@@ -514,7 +515,8 @@ dummy_func(
             _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
             ERROR_IF(*target_local == NULL, error);
             // The STORE_FAST is already done.
-            SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1);
+            assert(next_instr->op.code == STORE_FAST);
+            SKIP_OVER(1);
         }
 
         macro(BINARY_OP_INPLACE_ADD_UNICODE) =
@@ -530,14 +532,13 @@ dummy_func(
 
         inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) {
             #if ENABLE_SPECIALIZATION
-            _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_BinarySubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(BINARY_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             res = PyObject_GetItem(container, sub);
             DECREF_INPUTS();
@@ -656,8 +657,7 @@ dummy_func(
             STACK_SHRINK(2);
             new_frame->localsplus[0] = container;
             new_frame->localsplus[1] = sub;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
-            assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR;
             DISPATCH_INLINED(new_frame);
         }
@@ -679,14 +679,13 @@ dummy_func(
 
         inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) {
             #if ENABLE_SPECIALIZATION
-            _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_StoreSubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(STORE_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             /* container[sub] = v */
             int err = PyObject_SetItem(container, sub, v);
@@ -754,7 +753,7 @@ dummy_func(
             case 0:
                 if (do_raise(tstate, exc, cause)) {
                     assert(oparg == 0);
-                    monitor_reraise(tstate, frame, next_instr-1);
+                    monitor_reraise(tstate, frame, this_instr);
                     goto exception_unwind;
                 }
                 break;
@@ -808,7 +807,7 @@ dummy_func(
         inst(INSTRUMENTED_RETURN_VALUE, (retval --)) {
             int err = _Py_call_instrumentation_arg(
                     tstate, PY_MONITORING_EVENT_PY_RETURN,
-                    frame, next_instr-1, retval);
+                    frame, this_instr, retval);
             if (err) goto error;
             STACK_SHRINK(1);
             assert(EMPTY());
@@ -832,7 +831,7 @@ dummy_func(
             PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg);
             int err = _Py_call_instrumentation_arg(
                     tstate, PY_MONITORING_EVENT_PY_RETURN,
-                    frame, next_instr-1, retval);
+                    frame, this_instr, retval);
             if (err) goto error;
             Py_INCREF(retval);
             assert(EMPTY());
@@ -958,14 +957,13 @@ dummy_func(
 
         inst(SEND, (unused/1, receiver, v -- receiver, retval)) {
             #if ENABLE_SPECIALIZATION
-            _PySendCache *cache = (_PySendCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_Send(receiver, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(SEND, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert(frame != &entry_frame);
             if ((tstate->interp->eval_frame == NULL) &&
@@ -979,8 +977,7 @@ dummy_func(
                 gen->gi_frame_state = FRAME_EXECUTING;
                 gen->gi_exc_state.previous_item = tstate->exc_info;
                 tstate->exc_info = &gen->gi_exc_state;
-                SKIP_OVER(INLINE_CACHE_ENTRIES_SEND);
-                assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr);
+                assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr);
                 frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
                 DISPATCH_INLINED(gen_frame);
             }
@@ -993,7 +990,7 @@ dummy_func(
             if (retval == NULL) {
                 if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
                 ) {
-                    monitor_raise(tstate, frame, next_instr-1);
+                    monitor_raise(tstate, frame, this_instr);
                 }
                 if (_PyGen_FetchStopIterationValue(&retval) == 0) {
                     assert(retval != NULL);
@@ -1018,8 +1015,7 @@ dummy_func(
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_SEND);
-            assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr);
             frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
             DISPATCH_INLINED(gen_frame);
         }
@@ -1033,7 +1029,7 @@ dummy_func(
             _PyFrame_SetStackPointer(frame, stack_pointer - 1);
             int err = _Py_call_instrumentation_arg(
                     tstate, PY_MONITORING_EVENT_PY_YIELD,
-                    frame, next_instr-1, retval);
+                    frame, this_instr, retval);
             if (err) goto error;
             tstate->exc_info = gen->gi_exc_state.previous_item;
             gen->gi_exc_state.previous_item = NULL;
@@ -1093,7 +1089,7 @@ dummy_func(
             assert(exc && PyExceptionInstance_Check(exc));
             Py_INCREF(exc);
             _PyErr_SetRaisedException(tstate, exc);
-            monitor_reraise(tstate, frame, next_instr-1);
+            monitor_reraise(tstate, frame, this_instr);
             goto exception_unwind;
         }
 
@@ -1105,7 +1101,7 @@ dummy_func(
             else {
                 Py_INCREF(exc);
                 _PyErr_SetRaisedException(tstate, exc);
-                monitor_reraise(tstate, frame, next_instr-1);
+                monitor_reraise(tstate, frame, this_instr);
                 goto exception_unwind;
             }
         }
@@ -1120,7 +1116,7 @@ dummy_func(
             }
             else {
                 _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
-                monitor_reraise(tstate, frame, next_instr-1);
+                monitor_reraise(tstate, frame, this_instr);
                 goto exception_unwind;
             }
         }
@@ -1184,14 +1180,13 @@ dummy_func(
 
         inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) {
             #if ENABLE_SPECIALIZATION
-            _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(UNPACK_SEQUENCE, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject **top = stack_pointer + oparg - 1;
             int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
@@ -1247,15 +1242,14 @@ dummy_func(
 
         inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) {
             #if ENABLE_SPECIALIZATION
-            _PyAttrCache *cache = (_PyAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_StoreAttr(owner, next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(STORE_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(owner, name, v);
@@ -1369,15 +1363,14 @@ dummy_func(
 
         inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) {
             #if ENABLE_SPECIALIZATION
-            _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_GLOBAL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (PyDict_CheckExact(GLOBALS())
@@ -1692,11 +1685,10 @@ dummy_func(
             ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error);
         }
 
-        inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) {
-            _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr;
+        inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) {
             // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
             // don't want to specialize instrumented instructions
-            INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+            INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
         }
 
@@ -1706,24 +1698,24 @@ dummy_func(
         };
 
         inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) {
+            TIER_ONE_ONLY
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
-            int load_method = oparg & 1;
             #if ENABLE_SPECIALIZATION
-            _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            int load_method = oparg & 1;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_SUPER_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
 
             if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
                 PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
                 int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_CALL,
-                        frame, next_instr-1, global_super, arg);
+                        frame, this_instr, global_super, arg);
                 ERROR_IF(err, error);
             }
 
@@ -1736,12 +1728,12 @@ dummy_func(
                 if (super == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, global_super, arg);
+                        frame, this_instr, global_super, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, global_super, arg);
+                        frame, this_instr, global_super, arg);
                     if (err < 0) {
                         Py_CLEAR(super);
                     }
@@ -1819,15 +1811,14 @@ dummy_func(
 
         inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) {
             #if ENABLE_SPECIALIZATION
-            _PyAttrCache *cache = (_PyAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_LoadAttr(owner, next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
             if (oparg & 1) {
@@ -2018,8 +2009,7 @@ dummy_func(
             // Manipulate stack directly because we exit with DISPATCH_INLINED().
             STACK_SHRINK(1);
             new_frame->localsplus[0] = owner;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR);
-            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR;
             DISPATCH_INLINED(new_frame);
         }
@@ -2046,8 +2036,7 @@ dummy_func(
             STACK_SHRINK(1);
             new_frame->localsplus[0] = owner;
             new_frame->localsplus[1] = Py_NewRef(name);
-            SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR);
-            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR;
             DISPATCH_INLINED(new_frame);
         }
@@ -2142,14 +2131,13 @@ dummy_func(
 
         inst(COMPARE_OP, (unused/1, left, right -- res)) {
             #if ENABLE_SPECIALIZATION
-            _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_CompareOp(left, right, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(COMPARE_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert((oparg >> 5) <= Py_GE);
             res = PyObject_RichCompare(left, right, oparg >> 5);
@@ -2277,24 +2265,23 @@ dummy_func(
 
         inst(JUMP_BACKWARD, (--)) {
             CHECK_EVAL_BREAKER();
-            _Py_CODEUNIT *here = next_instr - 1;
             assert(oparg <= INSTR_OFFSET());
             JUMPBY(1-oparg);
             #if ENABLE_SPECIALIZATION
-            here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER);
-            if (here[1].cache > tstate->interp->optimizer_backedge_threshold &&
+            this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER);
+            if (this_instr[1].cache > tstate->interp->optimizer_backedge_threshold &&
                 // Double-check that the opcode isn't instrumented or something:
-                here->op.code == JUMP_BACKWARD)
+                this_instr->op.code == JUMP_BACKWARD)
             {
                 OPT_STAT_INC(attempts);
-                int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer);
+                int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer);
                 ERROR_IF(optimized < 0, error);
                 if (optimized) {
                     // Rewind and enter the executor:
-                    assert(here->op.code == ENTER_EXECUTOR);
-                    next_instr = here;
+                    assert(this_instr->op.code == ENTER_EXECUTOR);
+                    next_instr = this_instr;
                 }
-                here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1);
+                this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1);
             }
             #endif  /* ENABLE_SPECIALIZATION */
         }
@@ -2331,7 +2318,7 @@ dummy_func(
             assert(PyBool_Check(cond));
             int flag = Py_IsFalse(cond);
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
             JUMPBY(oparg * flag);
         }
@@ -2340,7 +2327,7 @@ dummy_func(
             assert(PyBool_Check(cond));
             int flag = Py_IsTrue(cond);
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
             JUMPBY(oparg * flag);
         }
@@ -2456,14 +2443,13 @@ dummy_func(
 
         inst(FOR_ITER, (unused/1, iter -- iter, next)) {
             #if ENABLE_SPECIALIZATION
-            _PyForIterCache *cache = (_PyForIterCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_ForIter(iter, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(FOR_ITER, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
             next = (*Py_TYPE(iter)->tp_iternext)(iter);
@@ -2472,15 +2458,14 @@ dummy_func(
                     if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
                         goto error;
                     }
-                    monitor_raise(tstate, frame, next_instr-1);
+                    monitor_raise(tstate, frame, this_instr);
                     _PyErr_Clear(tstate);
                 }
                 /* iterator ended normally */
-                assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR ||
-                       next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR);
+                assert(next_instr[oparg].op.code == END_FOR ||
+                       next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
                 Py_DECREF(iter);
                 STACK_SHRINK(1);
-                SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                 /* Jump forward oparg, then skip following END_FOR instruction */
                 JUMPBY(oparg + 1);
                 DISPATCH();
@@ -2488,32 +2473,31 @@ dummy_func(
             // Common case: no jump, leave it to the code generator
         }
 
-        inst(INSTRUMENTED_FOR_ITER, ( -- )) {
-            _Py_CODEUNIT *here = frame->instr_ptr;
+        inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) {
             _Py_CODEUNIT *target;
             PyObject *iter = TOP();
             PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
             if (next != NULL) {
                 PUSH(next);
-                target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER;
+                target = next_instr;
             }
             else {
                 if (_PyErr_Occurred(tstate)) {
                     if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
                         goto error;
                     }
-                    monitor_raise(tstate, frame, here);
+                    monitor_raise(tstate, frame, this_instr);
                     _PyErr_Clear(tstate);
                 }
                 /* iterator ended normally */
-                assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR ||
-                       next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR);
+                assert(next_instr[oparg].op.code == END_FOR ||
+                       next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
                 STACK_SHRINK(1);
                 Py_DECREF(iter);
                 /* Skip END_FOR */
-                target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1;
+                target = next_instr + oparg + 1;
             }
-            INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH);
+            INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH);
         }
 
         op(_ITER_CHECK_LIST, (iter -- iter)) {
@@ -2532,7 +2516,6 @@ dummy_func(
                 }
                 Py_DECREF(iter);
                 STACK_SHRINK(1);
-                SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                 /* Jump forward oparg, then skip following END_FOR instruction */
                 JUMPBY(oparg + 1);
                 DISPATCH();
@@ -2588,7 +2571,6 @@ dummy_func(
                 }
                 Py_DECREF(iter);
                 STACK_SHRINK(1);
-                SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                 /* Jump forward oparg, then skip following END_FOR instruction */
                 JUMPBY(oparg + 1);
                 DISPATCH();
@@ -2640,7 +2622,6 @@ dummy_func(
             if (r->len <= 0) {
                 STACK_SHRINK(1);
                 Py_DECREF(r);
-                SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                 // Jump over END_FOR instruction.
                 JUMPBY(oparg + 1);
                 DISPATCH();
@@ -2682,10 +2663,9 @@ dummy_func(
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
             assert(next_instr[oparg].op.code == END_FOR ||
                    next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
-            assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - this_instr);
             frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
             DISPATCH_INLINED(gen_frame);
         }
@@ -2912,7 +2892,7 @@ dummy_func(
             unused/2 +
             _LOAD_ATTR_METHOD_LAZY_DICT;
 
-        inst(INSTRUMENTED_CALL, ( -- )) {
+        inst(INSTRUMENTED_CALL, (unused/3 -- )) {
             int is_meth = PEEK(oparg + 1) != NULL;
             int total_args = oparg + is_meth;
             PyObject *function = PEEK(oparg + 2);
@@ -2920,10 +2900,9 @@ dummy_func(
                 &_PyInstrumentation_MISSING : PEEK(total_args);
             int err = _Py_call_instrumentation_2args(
                     tstate, PY_MONITORING_EVENT_CALL,
-                    frame, next_instr-1, function, arg);
+                    frame, this_instr, function, arg);
             ERROR_IF(err, error);
-            _PyCallCache *cache = (_PyCallCache *)next_instr;
-            INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+            INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             GO_TO_INSTRUCTION(CALL);
         }
 
@@ -2959,14 +2938,13 @@ dummy_func(
                 total_args++;
             }
             #if ENABLE_SPECIALIZATION
-            _PyCallCache *cache = (_PyCallCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_Call(callable, next_instr, total_args);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(CALL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
                 args--;
@@ -2996,8 +2974,7 @@ dummy_func(
                 if (new_frame == NULL) {
                     goto error;
                 }
-                SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
-                assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr);
+                assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
                 frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
                 DISPATCH_INLINED(new_frame);
             }
@@ -3012,12 +2989,12 @@ dummy_func(
                 if (res == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                     if (err < 0) {
                         Py_CLEAR(res);
                     }
@@ -3151,8 +3128,7 @@ dummy_func(
             }
             // Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
             STACK_SHRINK(oparg + 2);
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
-            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
             DISPATCH_INLINED(new_frame);
         }
@@ -3200,7 +3176,7 @@ dummy_func(
              * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
              * 3. Pushes the frame for ``__init__`` to the frame stack
              * */
-            _PyCallCache *cache = (_PyCallCache *)next_instr;
+            _PyCallCache *cache = (_PyCallCache *)&this_instr[1];
             DEOPT_IF(null != NULL);
             DEOPT_IF(!PyType_Check(callable));
             PyTypeObject *tp = (PyTypeObject *)callable;
@@ -3229,8 +3205,7 @@ dummy_func(
             for (int i = 0; i < oparg; i++) {
                 init_frame->localsplus[i+1] = args[i];
             }
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
-            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
             STACK_SHRINK(oparg+2);
             _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -3426,9 +3401,9 @@ dummy_func(
             Py_DECREF(self);
             Py_DECREF(callable);
             STACK_SHRINK(3);
-            // CALL + POP_TOP
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1);
-            assert(next_instr[-1].op.code == POP_TOP);
+            // Skip POP_TOP
+            assert(next_instr->op.code == POP_TOP);
+            SKIP_OVER(1);
             DISPATCH();
         }
 
@@ -3558,7 +3533,7 @@ dummy_func(
                                             : PEEK(total_args + 1);
             int err = _Py_call_instrumentation_2args(
                     tstate, PY_MONITORING_EVENT_CALL,
-                    frame, next_instr - 1, function, arg);
+                    frame, this_instr, function, arg);
             ERROR_IF(err, error);
             GO_TO_INSTRUCTION(CALL_KW);
         }
@@ -3600,7 +3575,7 @@ dummy_func(
                 if (new_frame == NULL) {
                     goto error;
                 }
-                assert(next_instr - frame->instr_ptr == 1);
+                assert(next_instr - this_instr == 1);
                 frame->return_offset = 1;
                 DISPATCH_INLINED(new_frame);
             }
@@ -3615,12 +3590,12 @@ dummy_func(
                 if (res == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                     if (err < 0) {
                         Py_CLEAR(res);
                     }
@@ -3663,18 +3638,18 @@ dummy_func(
                     PyTuple_GET_ITEM(callargs, 0) : Py_None;
                 int err = _Py_call_instrumentation_2args(
                     tstate, PY_MONITORING_EVENT_CALL,
-                    frame, next_instr-1, func, arg);
+                    frame, this_instr, func, arg);
                 if (err) goto error;
                 result = PyObject_Call(func, callargs, kwargs);
                 if (result == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, func, arg);
+                        frame, this_instr, func, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, func, arg);
+                        frame, this_instr, func, arg);
                     if (err < 0) {
                         Py_CLEAR(result);
                     }
@@ -3697,7 +3672,7 @@ dummy_func(
                     if (new_frame == NULL) {
                         goto error;
                     }
-                    assert(next_instr - frame->instr_ptr == 1);
+                    assert(next_instr - this_instr == 1);
                     frame->return_offset = 1;
                     DISPATCH_INLINED(new_frame);
                 }
@@ -3818,14 +3793,13 @@ dummy_func(
 
         inst(BINARY_OP, (unused/1, lhs, rhs -- res)) {
             #if ENABLE_SPECIALIZATION
-            _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(BINARY_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert(NB_ADD <= oparg);
             assert(oparg <= NB_INPLACE_XOR);
@@ -3842,12 +3816,11 @@ dummy_func(
 
         inst(INSTRUMENTED_INSTRUCTION, ( -- )) {
             int next_opcode = _Py_call_instrumentation_instruction(
-                tstate, frame, next_instr-1);
+                tstate, frame, this_instr);
             ERROR_IF(next_opcode < 0, error);
-            next_instr--;
+            next_instr = this_instr;
             if (_PyOpcode_Caches[next_opcode]) {
-                _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
-                INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+                INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             }
             assert(next_opcode > 0 && next_opcode < 256);
             opcode = next_opcode;
@@ -3855,43 +3828,38 @@ dummy_func(
         }
 
         inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) {
-            _Py_CODEUNIT *here = frame->instr_ptr;
-            INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
+            INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
         }
 
         inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) {
-            _Py_CODEUNIT *here = frame->instr_ptr;
             CHECK_EVAL_BREAKER();
-            INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP);
+            INSTRUMENTED_JUMP(this_instr, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP);
         }
 
         inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) {
             PyObject *cond = POP();
             assert(PyBool_Check(cond));
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int flag = Py_IsTrue(cond);
             int offset = flag * oparg;
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
         }
 
         inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) {
             PyObject *cond = POP();
             assert(PyBool_Check(cond));
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int flag = Py_IsFalse(cond);
             int offset = flag * oparg;
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
         }
 
         inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) {
             PyObject *value = POP();
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int flag = Py_IsNone(value);
             int offset;
             if (flag) {
@@ -3902,14 +3870,13 @@ dummy_func(
                 offset = 0;
             }
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
         }
 
         inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) {
             PyObject *value = POP();
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int offset;
             int nflag = Py_IsNone(value);
             if (nflag) {
@@ -3920,9 +3887,9 @@ dummy_func(
                 offset = oparg;
             }
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | !nflag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | !nflag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
         }
 
         inst(EXTENDED_ARG, ( -- )) {
@@ -3969,7 +3936,7 @@ dummy_func(
 
         op(_SAVE_RETURN_OFFSET, (--)) {
             #if TIER_ONE
-            frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr);
+            frame->return_offset = (uint16_t)(next_instr - this_instr);
             #endif
             #if TIER_TWO
             frame->return_offset = oparg;

diff --git a/Python/ceval.c b/Python/ceval.c
index 6f8584c15b7bde8f2db93ae74a5fe725b5d05fb8..bada365011226cd86957c212eeb8465cf7b4e044 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -867,7 +867,7 @@ error:
         monitor_raise(tstate, frame, next_instr-1);
 exception_unwind:
         {
-            /* We can't use frame->f_lasti here, as RERAISE may have set it */
+            /* We can't use frame->instr_ptr here, as RERAISE may have set it */
             int offset = INSTR_OFFSET()-1;
             int level, handler, lasti;
             if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) {

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 544e8ef8fa8c0ae9a6d0709878e309fc6881810e..b11240b1faed4930f2c47ac72d4535ae06da3292 100644 (file)
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -60,25 +60,21 @@
 #endif
 
 #ifdef Py_STATS
-#define INSTRUCTION_START(op) \
+#define INSTRUCTION_STATS(op) \
     do { \
-        frame->instr_ptr = next_instr++; \
         OPCODE_EXE_INC(op); \
         if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \
         lastopcode = op; \
     } while (0)
 #else
-#define INSTRUCTION_START(op) \
-    do { \
-        frame->instr_ptr = next_instr++; \
-    } while(0)
+#define INSTRUCTION_STATS(op) ((void)0)
 #endif
 
 #if USE_COMPUTED_GOTOS
-#  define TARGET(op) TARGET_##op: INSTRUCTION_START(op);
+#  define TARGET(op) TARGET_##op:
 #  define DISPATCH_GOTO() goto *opcode_targets[opcode]
 #else
-#  define TARGET(op) case op: TARGET_##op: INSTRUCTION_START(op);
+#  define TARGET(op) case op: TARGET_##op:
 #  define DISPATCH_GOTO() goto dispatch_opcode
 #endif
 

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 9f37cd75b47efb9743e5263ed258e8654eb07ac5..24de9eadf59048fedf323d66f21bf44558324265 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -119,14 +119,13 @@
             PyObject *res;
             value = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyToBoolCache *cache = (_PyToBoolCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_ToBool(value, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(TO_BOOL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             int err = PyObject_IsTrue(value);
             Py_DECREF(value);
@@ -380,14 +379,13 @@
             sub = stack_pointer[-1];
             container = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_BinarySubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(BINARY_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             res = PyObject_GetItem(container, sub);
             Py_DECREF(container);
@@ -574,14 +572,13 @@
             container = stack_pointer[-2];
             v = stack_pointer[-3];
             #if ENABLE_SPECIALIZATION
-            _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_StoreSubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(STORE_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             /* container[sub] = v */
             int err = PyObject_SetItem(container, sub, v);
@@ -900,14 +897,13 @@
             PyObject *seq;
             seq = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(UNPACK_SEQUENCE, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject **top = stack_pointer + oparg - 1;
             int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
@@ -989,15 +985,14 @@
             owner = stack_pointer[-1];
             v = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyAttrCache *cache = (_PyAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_StoreAttr(owner, next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(STORE_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(owner, name, v);
@@ -1133,15 +1128,14 @@
             PyObject *res;
             PyObject *null = NULL;
             #if ENABLE_SPECIALIZATION
-            _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_GLOBAL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (PyDict_CheckExact(GLOBALS())
@@ -1624,15 +1618,14 @@
             PyObject *self_or_null = NULL;
             owner = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyAttrCache *cache = (_PyAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_LoadAttr(owner, next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
             if (oparg & 1) {
@@ -1887,14 +1880,13 @@
             right = stack_pointer[-1];
             left = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_CompareOp(left, right, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(COMPARE_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert((oparg >> 5) <= Py_GE);
             res = PyObject_RichCompare(left, right, oparg >> 5);
@@ -3209,14 +3201,13 @@
             rhs = stack_pointer[-1];
             lhs = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(BINARY_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert(NB_ADD <= oparg);
             assert(oparg <= NB_INPLACE_XOR);
@@ -3275,7 +3266,7 @@
 
         case _SAVE_RETURN_OFFSET: {
             #if TIER_ONE
-            frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr);
+            frame->return_offset = (uint16_t)(next_instr - this_instr);
             #endif
             #if TIER_TWO
             frame->return_offset = oparg;

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index fb4506e68765ecf4b35eac401da3f2a0c9d2f452..40c8cd8798c93b694b0bbcaa0fcc8ed74384a358 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -4,11 +4,18 @@
 // Do not edit!
 
         TARGET(NOP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(NOP);
             DISPATCH();
         }
 
         TARGET(RESUME) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RESUME);
             PREDICTED(RESUME);
+            _Py_CODEUNIT *this_instr = next_instr - 1;
             static_assert(0 == 0, "incorrect cache size");
             TIER_ONE_ONLY
             assert(frame == tstate->current_frame);
@@ -20,18 +27,21 @@
             if (code_version != global_version) {
                 int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
                 if (err) goto error;
-                next_instr--;
+                next_instr = this_instr;
             }
             else {
                 if (oparg < RESUME_AFTER_YIELD_FROM) {
                     CHECK_EVAL_BREAKER();
                 }
-                next_instr[-1].op.code = RESUME_CHECK;
+                this_instr->op.code = RESUME_CHECK;
             }
             DISPATCH();
         }
 
         TARGET(RESUME_CHECK) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RESUME_CHECK);
 #if defined(__EMSCRIPTEN__)
             DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
             _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
@@ -44,13 +54,16 @@
         }
 
         TARGET(INSTRUMENTED_RESUME) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_RESUME);
             uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
             uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
             if (code_version != global_version) {
                 if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
                     goto error;
                 }
-                next_instr--;
+                next_instr = this_instr;
             }
             else {
                 if (oparg < 2) {
@@ -58,12 +71,12 @@
                 }
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 int err = _Py_call_instrumentation(
-                        tstate, oparg > 0, frame, next_instr-1);
+                        tstate, oparg > 0, frame, this_instr);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
                 if (err) goto error;
-                if (frame->instr_ptr != next_instr-1) {
+                if (frame->instr_ptr != this_instr) {
                     /* Instrumentation has jumped */
-                    next_instr = frame->instr_ptr;
+                    next_instr = this_instr;
                     DISPATCH();
                 }
             }
@@ -71,6 +84,9 @@
         }
 
         TARGET(LOAD_FAST_CHECK) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_FAST_CHECK);
             PyObject *value;
             value = GETLOCAL(oparg);
             if (value == NULL) goto unbound_local_error;
@@ -81,6 +97,9 @@
         }
 
         TARGET(LOAD_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_FAST);
             PyObject *value;
             value = GETLOCAL(oparg);
             assert(value != NULL);
@@ -91,6 +110,9 @@
         }
 
         TARGET(LOAD_FAST_AND_CLEAR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR);
             PyObject *value;
             value = GETLOCAL(oparg);
             // do not use SETLOCAL here, it decrefs the old value
@@ -101,6 +123,9 @@
         }
 
         TARGET(LOAD_FAST_LOAD_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST);
             PyObject *value1;
             PyObject *value2;
             uint32_t oparg1 = oparg >> 4;
@@ -116,6 +141,9 @@
         }
 
         TARGET(LOAD_CONST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_CONST);
             PyObject *value;
             value = GETITEM(FRAME_CO_CONSTS, oparg);
             Py_INCREF(value);
@@ -125,6 +153,9 @@
         }
 
         TARGET(STORE_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_FAST);
             PyObject *value;
             value = stack_pointer[-1];
             SETLOCAL(oparg, value);
@@ -133,6 +164,9 @@
         }
 
         TARGET(STORE_FAST_LOAD_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_FAST_LOAD_FAST);
             PyObject *value1;
             PyObject *value2;
             value1 = stack_pointer[-1];
@@ -146,6 +180,9 @@
         }
 
         TARGET(STORE_FAST_STORE_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_FAST_STORE_FAST);
             PyObject *value1;
             PyObject *value2;
             value1 = stack_pointer[-1];
@@ -159,6 +196,9 @@
         }
 
         TARGET(POP_TOP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(POP_TOP);
             PyObject *value;
             value = stack_pointer[-1];
             Py_DECREF(value);
@@ -167,6 +207,9 @@
         }
 
         TARGET(PUSH_NULL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(PUSH_NULL);
             PyObject *res;
             res = NULL;
             STACK_GROW(1);
@@ -175,6 +218,9 @@
         }
 
         TARGET(END_FOR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(END_FOR);
             PyObject *value;
             // POP_TOP
             value = stack_pointer[-1];
@@ -191,15 +237,19 @@
         }
 
         TARGET(INSTRUMENTED_END_FOR) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_END_FOR);
             PyObject *value;
             PyObject *receiver;
             value = stack_pointer[-1];
             receiver = stack_pointer[-2];
+            TIER_ONE_ONLY
             /* Need to create a fake StopIteration error here,
              * to conform to PEP 380 */
             if (PyGen_Check(receiver)) {
                 PyErr_SetObject(PyExc_StopIteration, value);
-                if (monitor_stop_iteration(tstate, frame, next_instr-1)) {
+                if (monitor_stop_iteration(tstate, frame, this_instr)) {
                     goto error;
                 }
                 PyErr_SetRaisedException(NULL);
@@ -211,6 +261,9 @@
         }
 
         TARGET(END_SEND) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(END_SEND);
             PyObject *value;
             PyObject *receiver;
             value = stack_pointer[-1];
@@ -222,13 +275,17 @@
         }
 
         TARGET(INSTRUMENTED_END_SEND) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_END_SEND);
             PyObject *value;
             PyObject *receiver;
             value = stack_pointer[-1];
             receiver = stack_pointer[-2];
+            TIER_ONE_ONLY
             if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
                 PyErr_SetObject(PyExc_StopIteration, value);
-                if (monitor_stop_iteration(tstate, frame, next_instr-1)) {
+                if (monitor_stop_iteration(tstate, frame, this_instr)) {
                     goto error;
                 }
                 PyErr_SetRaisedException(NULL);
@@ -240,6 +297,9 @@
         }
 
         TARGET(UNARY_NEGATIVE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(UNARY_NEGATIVE);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -251,6 +311,9 @@
         }
 
         TARGET(UNARY_NOT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(UNARY_NOT);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -261,40 +324,47 @@
         }
 
         TARGET(TO_BOOL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL);
             PREDICTED(TO_BOOL);
+            _Py_CODEUNIT *this_instr = next_instr - 4;
             static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyToBoolCache *cache = (_PyToBoolCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_ToBool(value, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(TO_BOOL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             int err = PyObject_IsTrue(value);
             Py_DECREF(value);
             if (err < 0) goto pop_1_error;
             res = err ? Py_True : Py_False;
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(TO_BOOL_BOOL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL_BOOL);
             PyObject *value;
             value = stack_pointer[-1];
             DEOPT_IF(!PyBool_Check(value), TO_BOOL);
             STAT_INC(TO_BOOL, hit);
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(TO_BOOL_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL_INT);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -309,11 +379,13 @@
                 res = Py_True;
             }
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(TO_BOOL_LIST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL_LIST);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -322,11 +394,13 @@
             res = Py_SIZE(value) ? Py_True : Py_False;
             Py_DECREF(value);
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(TO_BOOL_NONE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL_NONE);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -335,11 +409,13 @@
             STAT_INC(TO_BOOL, hit);
             res = Py_False;
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(TO_BOOL_STR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL_STR);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -355,15 +431,17 @@
                 res = Py_True;
             }
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(TO_BOOL_ALWAYS_TRUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
-            uint32_t version = read_u32(&next_instr[1].cache);
+            uint32_t version = read_u32(&this_instr[2].cache);
             // This one is a bit weird, because we expect *some* failures:
             assert(version);
             DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
@@ -371,11 +449,13 @@
             Py_DECREF(value);
             res = Py_True;
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(UNARY_INVERT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(UNARY_INVERT);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -387,6 +467,9 @@
         }
 
         TARGET(BINARY_OP_MULTIPLY_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -407,11 +490,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_ADD_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_ADD_INT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -432,11 +517,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_SUBTRACT_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -457,11 +544,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_MULTIPLY_FLOAT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -482,11 +571,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_ADD_FLOAT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -507,11 +598,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_SUBTRACT_FLOAT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -532,11 +625,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_ADD_UNICODE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -557,11 +652,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
             PyObject *right;
             PyObject *left;
             // _GUARD_BOTH_UNICODE
@@ -573,9 +670,8 @@
             }
             // _BINARY_OP_INPLACE_ADD_UNICODE
             {
-                _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
-                assert(true_next.op.code == STORE_FAST);
-                PyObject **target_local = &GETLOCAL(true_next.op.arg);
+                assert(next_instr->op.code == STORE_FAST);
+                PyObject **target_local = &GETLOCAL(next_instr->op.arg);
                 DEOPT_IF(*target_local != left, BINARY_OP);
                 STAT_INC(BINARY_OP, hit);
                 /* Handle `left = left + right` or `left += right` for str.
@@ -595,14 +691,19 @@
                 _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
                 if (*target_local == NULL) goto pop_2_error;
                 // The STORE_FAST is already done.
-                SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1);
+                assert(next_instr->op.code == STORE_FAST);
+                SKIP_OVER(1);
             }
             STACK_SHRINK(2);
             DISPATCH();
         }
 
         TARGET(BINARY_SUBSCR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_SUBSCR);
             PREDICTED(BINARY_SUBSCR);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
             PyObject *sub;
             PyObject *container;
@@ -610,14 +711,13 @@
             sub = stack_pointer[-1];
             container = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_BinarySubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(BINARY_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             res = PyObject_GetItem(container, sub);
             Py_DECREF(container);
@@ -625,11 +725,13 @@
             if (res == NULL) goto pop_2_error;
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_SLICE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BINARY_SLICE);
             PyObject *stop;
             PyObject *start;
             PyObject *container;
@@ -655,6 +757,9 @@
         }
 
         TARGET(STORE_SLICE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_SLICE);
             PyObject *stop;
             PyObject *start;
             PyObject *container;
@@ -680,6 +785,9 @@
         }
 
         TARGET(BINARY_SUBSCR_LIST_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
             PyObject *sub;
             PyObject *list;
             PyObject *res;
@@ -700,11 +808,13 @@
             Py_DECREF(list);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_SUBSCR_STR_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
             PyObject *sub;
             PyObject *str;
             PyObject *res;
@@ -724,11 +834,13 @@
             Py_DECREF(str);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_SUBSCR_TUPLE_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
             PyObject *sub;
             PyObject *tuple;
             PyObject *res;
@@ -749,11 +861,13 @@
             Py_DECREF(tuple);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_SUBSCR_DICT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
             PyObject *sub;
             PyObject *dict;
             PyObject *res;
@@ -775,11 +889,13 @@
             Py_DECREF(sub);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(BINARY_SUBSCR_GETITEM) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
             PyObject *sub;
             PyObject *container;
             sub = stack_pointer[-1];
@@ -803,13 +919,15 @@
             STACK_SHRINK(2);
             new_frame->localsplus[0] = container;
             new_frame->localsplus[1] = sub;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
-            assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR;
             DISPATCH_INLINED(new_frame);
         }
 
         TARGET(LIST_APPEND) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LIST_APPEND);
             PyObject *v;
             PyObject *list;
             v = stack_pointer[-1];
@@ -820,6 +938,9 @@
         }
 
         TARGET(SET_ADD) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(SET_ADD);
             PyObject *v;
             PyObject *set;
             v = stack_pointer[-1];
@@ -832,7 +953,11 @@
         }
 
         TARGET(STORE_SUBSCR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(STORE_SUBSCR);
             PREDICTED(STORE_SUBSCR);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
             PyObject *sub;
             PyObject *container;
@@ -841,14 +966,13 @@
             container = stack_pointer[-2];
             v = stack_pointer[-3];
             #if ENABLE_SPECIALIZATION
-            _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_StoreSubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(STORE_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             /* container[sub] = v */
             int err = PyObject_SetItem(container, sub, v);
@@ -857,11 +981,13 @@
             Py_DECREF(sub);
             if (err) goto pop_3_error;
             STACK_SHRINK(3);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(STORE_SUBSCR_LIST_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
             PyObject *sub;
             PyObject *list;
             PyObject *value;
@@ -885,11 +1011,13 @@
             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
             Py_DECREF(list);
             STACK_SHRINK(3);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(STORE_SUBSCR_DICT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(STORE_SUBSCR_DICT);
             PyObject *sub;
             PyObject *dict;
             PyObject *value;
@@ -902,11 +1030,13 @@
             Py_DECREF(dict);
             if (err) goto pop_3_error;
             STACK_SHRINK(3);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(DELETE_SUBSCR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DELETE_SUBSCR);
             PyObject *sub;
             PyObject *container;
             sub = stack_pointer[-1];
@@ -921,6 +1051,9 @@
         }
 
         TARGET(CALL_INTRINSIC_1) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CALL_INTRINSIC_1);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -933,6 +1066,9 @@
         }
 
         TARGET(CALL_INTRINSIC_2) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CALL_INTRINSIC_2);
             PyObject *value1;
             PyObject *value2;
             PyObject *res;
@@ -949,6 +1085,9 @@
         }
 
         TARGET(RAISE_VARARGS) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RAISE_VARARGS);
             PyObject **args;
             args = stack_pointer - oparg;
             PyObject *cause = NULL, *exc = NULL;
@@ -962,7 +1101,7 @@
             case 0:
                 if (do_raise(tstate, exc, cause)) {
                     assert(oparg == 0);
-                    monitor_reraise(tstate, frame, next_instr-1);
+                    monitor_reraise(tstate, frame, this_instr);
                     goto exception_unwind;
                 }
                 break;
@@ -975,6 +1114,9 @@
         }
 
         TARGET(INTERPRETER_EXIT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INTERPRETER_EXIT);
             PyObject *retval;
             retval = stack_pointer[-1];
             assert(frame == &entry_frame);
@@ -987,6 +1129,9 @@
         }
 
         TARGET(RETURN_VALUE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RETURN_VALUE);
             PyObject *retval;
             retval = stack_pointer[-1];
             STACK_SHRINK(1);
@@ -1013,11 +1158,14 @@
         }
 
         TARGET(INSTRUMENTED_RETURN_VALUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE);
             PyObject *retval;
             retval = stack_pointer[-1];
             int err = _Py_call_instrumentation_arg(
                     tstate, PY_MONITORING_EVENT_PY_RETURN,
-                    frame, next_instr-1, retval);
+                    frame, this_instr, retval);
             if (err) goto error;
             STACK_SHRINK(1);
             assert(EMPTY());
@@ -1034,6 +1182,9 @@
         }
 
         TARGET(RETURN_CONST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RETURN_CONST);
             PyObject *value;
             PyObject *retval;
             // LOAD_CONST
@@ -1068,10 +1219,13 @@
         }
 
         TARGET(INSTRUMENTED_RETURN_CONST) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST);
             PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg);
             int err = _Py_call_instrumentation_arg(
                     tstate, PY_MONITORING_EVENT_PY_RETURN,
-                    frame, next_instr-1, retval);
+                    frame, this_instr, retval);
             if (err) goto error;
             Py_INCREF(retval);
             assert(EMPTY());
@@ -1088,6 +1242,9 @@
         }
 
         TARGET(GET_AITER) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(GET_AITER);
             PyObject *obj;
             PyObject *iter;
             obj = stack_pointer[-1];
@@ -1126,6 +1283,9 @@
         }
 
         TARGET(GET_ANEXT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(GET_ANEXT);
             PyObject *aiter;
             PyObject *awaitable;
             aiter = stack_pointer[-1];
@@ -1177,6 +1337,9 @@
         }
 
         TARGET(GET_AWAITABLE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(GET_AWAITABLE);
             PyObject *iterable;
             PyObject *iter;
             iterable = stack_pointer[-1];
@@ -1208,7 +1371,11 @@
         }
 
         TARGET(SEND) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(SEND);
             PREDICTED(SEND);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
             PyObject *v;
             PyObject *receiver;
@@ -1216,14 +1383,13 @@
             v = stack_pointer[-1];
             receiver = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PySendCache *cache = (_PySendCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_Send(receiver, next_instr);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(SEND, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert(frame != &entry_frame);
             if ((tstate->interp->eval_frame == NULL) &&
@@ -1237,8 +1403,7 @@
                 gen->gi_frame_state = FRAME_EXECUTING;
                 gen->gi_exc_state.previous_item = tstate->exc_info;
                 tstate->exc_info = &gen->gi_exc_state;
-                SKIP_OVER(INLINE_CACHE_ENTRIES_SEND);
-                assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr);
+                assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr);
                 frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
                 DISPATCH_INLINED(gen_frame);
             }
@@ -1251,7 +1416,7 @@
             if (retval == NULL) {
                 if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
                 ) {
-                    monitor_raise(tstate, frame, next_instr-1);
+                    monitor_raise(tstate, frame, this_instr);
                 }
                 if (_PyGen_FetchStopIterationValue(&retval) == 0) {
                     assert(retval != NULL);
@@ -1263,11 +1428,13 @@
             }
             Py_DECREF(v);
             stack_pointer[-1] = retval;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(SEND_GEN) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(SEND_GEN);
             PyObject *v;
             PyObject *receiver;
             v = stack_pointer[-1];
@@ -1283,13 +1450,15 @@
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_SEND);
-            assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr);
             frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
             DISPATCH_INLINED(gen_frame);
         }
 
         TARGET(INSTRUMENTED_YIELD_VALUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE);
             PyObject *retval;
             retval = stack_pointer[-1];
             assert(frame != &entry_frame);
@@ -1300,7 +1469,7 @@
             _PyFrame_SetStackPointer(frame, stack_pointer - 1);
             int err = _Py_call_instrumentation_arg(
                     tstate, PY_MONITORING_EVENT_PY_YIELD,
-                    frame, next_instr-1, retval);
+                    frame, this_instr, retval);
             if (err) goto error;
             tstate->exc_info = gen->gi_exc_state.previous_item;
             gen->gi_exc_state.previous_item = NULL;
@@ -1316,6 +1485,9 @@
         }
 
         TARGET(YIELD_VALUE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(YIELD_VALUE);
             PyObject *retval;
             retval = stack_pointer[-1];
             // NOTE: It's important that YIELD_VALUE never raises an exception!
@@ -1341,6 +1513,9 @@
         }
 
         TARGET(POP_EXCEPT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(POP_EXCEPT);
             PyObject *exc_value;
             exc_value = stack_pointer[-1];
             _PyErr_StackItem *exc_info = tstate->exc_info;
@@ -1350,6 +1525,9 @@
         }
 
         TARGET(RERAISE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RERAISE);
             PyObject *exc;
             PyObject **values;
             exc = stack_pointer[-1];
@@ -1370,11 +1548,14 @@
             assert(exc && PyExceptionInstance_Check(exc));
             Py_INCREF(exc);
             _PyErr_SetRaisedException(tstate, exc);
-            monitor_reraise(tstate, frame, next_instr-1);
+            monitor_reraise(tstate, frame, this_instr);
             goto exception_unwind;
         }
 
         TARGET(END_ASYNC_FOR) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(END_ASYNC_FOR);
             PyObject *exc;
             PyObject *awaitable;
             exc = stack_pointer[-1];
@@ -1387,7 +1568,7 @@
             else {
                 Py_INCREF(exc);
                 _PyErr_SetRaisedException(tstate, exc);
-                monitor_reraise(tstate, frame, next_instr-1);
+                monitor_reraise(tstate, frame, this_instr);
                 goto exception_unwind;
             }
             STACK_SHRINK(2);
@@ -1395,6 +1576,9 @@
         }
 
         TARGET(CLEANUP_THROW) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CLEANUP_THROW);
             PyObject *exc_value;
             PyObject *last_sent_val;
             PyObject *sub_iter;
@@ -1414,7 +1598,7 @@
             }
             else {
                 _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
-                monitor_reraise(tstate, frame, next_instr-1);
+                monitor_reraise(tstate, frame, this_instr);
                 goto exception_unwind;
             }
             STACK_SHRINK(1);
@@ -1424,6 +1608,9 @@
         }
 
         TARGET(LOAD_ASSERTION_ERROR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_ASSERTION_ERROR);
             PyObject *value;
             value = Py_NewRef(PyExc_AssertionError);
             STACK_GROW(1);
@@ -1432,6 +1619,9 @@
         }
 
         TARGET(LOAD_BUILD_CLASS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_BUILD_CLASS);
             PyObject *bc;
             if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error;
             if (bc == NULL) {
@@ -1445,6 +1635,9 @@
         }
 
         TARGET(STORE_NAME) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_NAME);
             PyObject *v;
             v = stack_pointer[-1];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
@@ -1467,6 +1660,9 @@
         }
 
         TARGET(DELETE_NAME) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DELETE_NAME);
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             PyObject *ns = LOCALS();
             int err;
@@ -1487,19 +1683,22 @@
         }
 
         TARGET(UNPACK_SEQUENCE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(UNPACK_SEQUENCE);
             PREDICTED(UNPACK_SEQUENCE);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
             PyObject *seq;
             seq = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(UNPACK_SEQUENCE, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject **top = stack_pointer + oparg - 1;
             int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
@@ -1507,11 +1706,13 @@
             if (res == 0) goto pop_1_error;
             STACK_SHRINK(1);
             STACK_GROW(oparg);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
             PyObject *seq;
             PyObject **values;
             seq = stack_pointer[-1];
@@ -1525,11 +1726,13 @@
             Py_DECREF(seq);
             STACK_SHRINK(1);
             STACK_GROW(oparg);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(UNPACK_SEQUENCE_TUPLE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
             PyObject *seq;
             PyObject **values;
             seq = stack_pointer[-1];
@@ -1544,11 +1747,13 @@
             Py_DECREF(seq);
             STACK_SHRINK(1);
             STACK_GROW(oparg);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(UNPACK_SEQUENCE_LIST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
             PyObject *seq;
             PyObject **values;
             seq = stack_pointer[-1];
@@ -1563,11 +1768,13 @@
             Py_DECREF(seq);
             STACK_SHRINK(1);
             STACK_GROW(oparg);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(UNPACK_EX) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(UNPACK_EX);
             PyObject *seq;
             seq = stack_pointer[-1];
             int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
@@ -1580,22 +1787,25 @@
         }
 
         TARGET(STORE_ATTR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(STORE_ATTR);
             PREDICTED(STORE_ATTR);
+            _Py_CODEUNIT *this_instr = next_instr - 5;
             static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
             PyObject *owner;
             PyObject *v;
             owner = stack_pointer[-1];
             v = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyAttrCache *cache = (_PyAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_StoreAttr(owner, next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(STORE_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(owner, name, v);
@@ -1603,11 +1813,13 @@
             Py_DECREF(owner);
             if (err) goto pop_2_error;
             STACK_SHRINK(2);
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(DELETE_ATTR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DELETE_ATTR);
             PyObject *owner;
             owner = stack_pointer[-1];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
@@ -1619,6 +1831,9 @@
         }
 
         TARGET(STORE_GLOBAL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_GLOBAL);
             PyObject *v;
             v = stack_pointer[-1];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
@@ -1630,6 +1845,9 @@
         }
 
         TARGET(DELETE_GLOBAL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DELETE_GLOBAL);
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err;
             err = PyDict_DelItem(GLOBALS(), name);
@@ -1645,6 +1863,9 @@
         }
 
         TARGET(LOAD_LOCALS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_LOCALS);
             PyObject *locals;
             locals = LOCALS();
             if (locals == NULL) {
@@ -1659,6 +1880,9 @@
         }
 
         TARGET(LOAD_FROM_DICT_OR_GLOBALS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS);
             PyObject *mod_or_class_dict;
             PyObject *v;
             mod_or_class_dict = stack_pointer[-1];
@@ -1692,6 +1916,9 @@
         }
 
         TARGET(LOAD_NAME) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_NAME);
             PyObject *v;
             PyObject *mod_or_class_dict = LOCALS();
             if (mod_or_class_dict == NULL) {
@@ -1729,20 +1956,23 @@
         }
 
         TARGET(LOAD_GLOBAL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(LOAD_GLOBAL);
             PREDICTED(LOAD_GLOBAL);
+            _Py_CODEUNIT *this_instr = next_instr - 5;
             static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
             PyObject *res;
             PyObject *null = NULL;
             #if ENABLE_SPECIALIZATION
-            _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_GLOBAL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (PyDict_CheckExact(GLOBALS())
@@ -1783,16 +2013,18 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(LOAD_GLOBAL_MODULE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
             PyObject *res;
             PyObject *null = NULL;
             // _GUARD_GLOBALS_VERSION
             {
-                uint16_t version = read_u16(&next_instr[1].cache);
+                uint16_t version = read_u16(&this_instr[2].cache);
                 PyDictObject *dict = (PyDictObject *)GLOBALS();
                 DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
                 DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
@@ -1800,7 +2032,7 @@
             }
             // _LOAD_GLOBAL_MODULE
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 PyDictObject *dict = (PyDictObject *)GLOBALS();
                 PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
                 res = entries[index].me_value;
@@ -1813,16 +2045,18 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(LOAD_GLOBAL_BUILTIN) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
             PyObject *res;
             PyObject *null = NULL;
             // _GUARD_GLOBALS_VERSION
             {
-                uint16_t version = read_u16(&next_instr[1].cache);
+                uint16_t version = read_u16(&this_instr[2].cache);
                 PyDictObject *dict = (PyDictObject *)GLOBALS();
                 DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
                 DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
@@ -1830,7 +2064,7 @@
             }
             // _GUARD_BUILTINS_VERSION
             {
-                uint16_t version = read_u16(&next_instr[2].cache);
+                uint16_t version = read_u16(&this_instr[3].cache);
                 PyDictObject *dict = (PyDictObject *)BUILTINS();
                 DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
                 DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
@@ -1838,7 +2072,7 @@
             }
             // _LOAD_GLOBAL_BUILTINS
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 PyDictObject *bdict = (PyDictObject *)BUILTINS();
                 PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
                 res = entries[index].me_value;
@@ -1851,11 +2085,13 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(DELETE_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DELETE_FAST);
             PyObject *v = GETLOCAL(oparg);
             if (v == NULL) goto unbound_local_error;
             SETLOCAL(oparg, NULL);
@@ -1863,6 +2099,9 @@
         }
 
         TARGET(MAKE_CELL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MAKE_CELL);
             // "initial" is probably NULL but not if it's an arg (or set
             // via PyFrame_LocalsToFast() before MAKE_CELL has run).
             PyObject *initial = GETLOCAL(oparg);
@@ -1875,6 +2114,9 @@
         }
 
         TARGET(DELETE_DEREF) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DELETE_DEREF);
             PyObject *cell = GETLOCAL(oparg);
             PyObject *oldobj = PyCell_GET(cell);
             // Can't use ERROR_IF here.
@@ -1889,6 +2131,9 @@
         }
 
         TARGET(LOAD_FROM_DICT_OR_DEREF) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF);
             PyObject *class_dict;
             PyObject *value;
             class_dict = stack_pointer[-1];
@@ -1915,6 +2160,9 @@
         }
 
         TARGET(LOAD_DEREF) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_DEREF);
             PyObject *value;
             PyObject *cell = GETLOCAL(oparg);
             value = PyCell_GET(cell);
@@ -1929,6 +2177,9 @@
         }
 
         TARGET(STORE_DEREF) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(STORE_DEREF);
             PyObject *v;
             v = stack_pointer[-1];
             PyObject *cell = GETLOCAL(oparg);
@@ -1940,6 +2191,9 @@
         }
 
         TARGET(COPY_FREE_VARS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(COPY_FREE_VARS);
             /* Copy closure variables to free variables */
             PyCodeObject *co = _PyFrame_GetCode(frame);
             assert(PyFunction_Check(frame->f_funcobj));
@@ -1954,6 +2208,9 @@
         }
 
         TARGET(BUILD_STRING) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_STRING);
             PyObject **pieces;
             PyObject *str;
             pieces = stack_pointer - oparg;
@@ -1969,6 +2226,9 @@
         }
 
         TARGET(BUILD_TUPLE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_TUPLE);
             PyObject **values;
             PyObject *tup;
             values = stack_pointer - oparg;
@@ -1981,6 +2241,9 @@
         }
 
         TARGET(BUILD_LIST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_LIST);
             PyObject **values;
             PyObject *list;
             values = stack_pointer - oparg;
@@ -1993,6 +2256,9 @@
         }
 
         TARGET(LIST_EXTEND) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LIST_EXTEND);
             PyObject *iterable;
             PyObject *list;
             iterable = stack_pointer[-1];
@@ -2017,6 +2283,9 @@
         }
 
         TARGET(SET_UPDATE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(SET_UPDATE);
             PyObject *iterable;
             PyObject *set;
             iterable = stack_pointer[-1];
@@ -2029,6 +2298,9 @@
         }
 
         TARGET(BUILD_SET) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_SET);
             PyObject **values;
             PyObject *set;
             values = stack_pointer - oparg;
@@ -2053,6 +2325,9 @@
         }
 
         TARGET(BUILD_MAP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_MAP);
             PyObject **values;
             PyObject *map;
             values = stack_pointer - oparg*2;
@@ -2071,6 +2346,9 @@
         }
 
         TARGET(SETUP_ANNOTATIONS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(SETUP_ANNOTATIONS);
             int err;
             PyObject *ann_dict;
             if (LOCALS() == NULL) {
@@ -2112,6 +2390,9 @@
         }
 
         TARGET(BUILD_CONST_KEY_MAP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_CONST_KEY_MAP);
             PyObject *keys;
             PyObject **values;
             PyObject *map;
@@ -2137,6 +2418,9 @@
         }
 
         TARGET(DICT_UPDATE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DICT_UPDATE);
             PyObject *update;
             PyObject *dict;
             update = stack_pointer[-1];
@@ -2156,6 +2440,9 @@
         }
 
         TARGET(DICT_MERGE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(DICT_MERGE);
             PyObject *update;
             PyObject *dict;
             PyObject *callable;
@@ -2173,6 +2460,9 @@
         }
 
         TARGET(MAP_ADD) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MAP_ADD);
             PyObject *value;
             PyObject *key;
             PyObject *dict;
@@ -2188,15 +2478,21 @@
         }
 
         TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) {
-            _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr;
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR);
             // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
             // don't want to specialize instrumented instructions
-            INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+            INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
         }
 
         TARGET(LOAD_SUPER_ATTR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(LOAD_SUPER_ATTR);
             PREDICTED(LOAD_SUPER_ATTR);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
             PyObject *self;
             PyObject *class;
@@ -2206,24 +2502,24 @@
             self = stack_pointer[-1];
             class = stack_pointer[-2];
             global_super = stack_pointer[-3];
+            TIER_ONE_ONLY
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
-            int load_method = oparg & 1;
             #if ENABLE_SPECIALIZATION
-            _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            int load_method = oparg & 1;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_SUPER_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
 
             if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
                 PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
                 int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_CALL,
-                        frame, next_instr-1, global_super, arg);
+                        frame, this_instr, global_super, arg);
                 if (err) goto pop_3_error;
             }
 
@@ -2236,12 +2532,12 @@
                 if (super == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, global_super, arg);
+                        frame, this_instr, global_super, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, global_super, arg);
+                        frame, this_instr, global_super, arg);
                     if (err < 0) {
                         Py_CLEAR(super);
                     }
@@ -2259,11 +2555,13 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(LOAD_SUPER_ATTR_ATTR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
             PyObject *self;
             PyObject *class;
             PyObject *global_super;
@@ -2283,11 +2581,13 @@
             if (attr == NULL) goto pop_3_error;
             STACK_SHRINK(2);
             stack_pointer[-1] = attr;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(LOAD_SUPER_ATTR_METHOD) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
             PyObject *self;
             PyObject *class;
             PyObject *global_super;
@@ -2320,27 +2620,29 @@
             STACK_SHRINK(1);
             stack_pointer[-2] = attr;
             stack_pointer[-1] = self_or_null;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR);
             PREDICTED(LOAD_ATTR);
+            _Py_CODEUNIT *this_instr = next_instr - 10;
             static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
             PyObject *owner;
             PyObject *attr;
             PyObject *self_or_null = NULL;
             owner = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyAttrCache *cache = (_PyAttrCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr--;
+                next_instr = this_instr;
                 _Py_Specialize_LoadAttr(owner, next_instr, name);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(LOAD_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
             if (oparg & 1) {
@@ -2377,18 +2679,20 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; }
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_INSTANCE_VALUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -2402,7 +2706,7 @@
             }
             // _LOAD_ATTR_INSTANCE_VALUE
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
                 attr = _PyDictOrValues_GetValues(dorv)->values[index];
                 DEOPT_IF(attr == NULL, LOAD_ATTR);
@@ -2414,18 +2718,20 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_MODULE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_MODULE);
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             // _CHECK_ATTR_MODULE
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
                 PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
                 assert(dict != NULL);
@@ -2433,7 +2739,7 @@
             }
             // _LOAD_ATTR_MODULE
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
                 assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
                 assert(index < dict->ma_keys->dk_nentries);
@@ -2448,18 +2754,20 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_WITH_HINT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -2475,7 +2783,7 @@
             }
             // _LOAD_ATTR_WITH_HINT
             {
-                uint16_t hint = read_u16(&next_instr[3].cache);
+                uint16_t hint = read_u16(&this_instr[4].cache);
                 PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
                 PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
                 DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
@@ -2499,25 +2807,27 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_SLOT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_SLOT);
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
             }
             // _LOAD_ATTR_SLOT
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 char *addr = (char *)owner + index;
                 attr = *(PyObject **)addr;
                 DEOPT_IF(attr == NULL, LOAD_ATTR);
@@ -2529,25 +2839,27 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_CLASS) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_CLASS);
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
             // _CHECK_ATTR_CLASS
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 DEOPT_IF(!PyType_Check(owner), LOAD_ATTR);
                 assert(type_version != 0);
                 DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR);
             }
             // _LOAD_ATTR_CLASS
             {
-                PyObject *descr = read_obj(&next_instr[5].cache);
+                PyObject *descr = read_obj(&this_instr[6].cache);
                 STAT_INC(LOAD_ATTR, hit);
                 assert(descr != NULL);
                 attr = Py_NewRef(descr);
@@ -2557,16 +2869,18 @@
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_PROPERTY) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
             PyObject *owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = read_u32(&next_instr[1].cache);
-            uint32_t func_version = read_u32(&next_instr[3].cache);
-            PyObject *fget = read_obj(&next_instr[5].cache);
+            uint32_t type_version = read_u32(&this_instr[2].cache);
+            uint32_t func_version = read_u32(&this_instr[4].cache);
+            PyObject *fget = read_obj(&this_instr[6].cache);
             assert((oparg & 1) == 0);
             DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
 
@@ -2586,18 +2900,20 @@
             // Manipulate stack directly because we exit with DISPATCH_INLINED().
             STACK_SHRINK(1);
             new_frame->localsplus[0] = owner;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR);
-            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR;
             DISPATCH_INLINED(new_frame);
         }
 
         TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
             PyObject *owner;
             owner = stack_pointer[-1];
-            uint32_t type_version = read_u32(&next_instr[1].cache);
-            uint32_t func_version = read_u32(&next_instr[3].cache);
-            PyObject *getattribute = read_obj(&next_instr[5].cache);
+            uint32_t type_version = read_u32(&this_instr[2].cache);
+            uint32_t func_version = read_u32(&this_instr[4].cache);
+            PyObject *getattribute = read_obj(&this_instr[6].cache);
             assert((oparg & 1) == 0);
             DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
             PyTypeObject *cls = Py_TYPE(owner);
@@ -2619,19 +2935,21 @@
             STACK_SHRINK(1);
             new_frame->localsplus[0] = owner;
             new_frame->localsplus[1] = Py_NewRef(name);
-            SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR);
-            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR;
             DISPATCH_INLINED(new_frame);
         }
 
         TARGET(STORE_ATTR_INSTANCE_VALUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
             PyObject *owner;
             PyObject *value;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
@@ -2645,7 +2963,7 @@
             // _STORE_ATTR_INSTANCE_VALUE
             value = stack_pointer[-2];
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
                 STAT_INC(STORE_ATTR, hit);
                 PyDictValues *values = _PyDictOrValues_GetValues(dorv);
@@ -2660,17 +2978,19 @@
                 Py_DECREF(owner);
             }
             STACK_SHRINK(2);
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(STORE_ATTR_WITH_HINT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
             PyObject *owner;
             PyObject *value;
             owner = stack_pointer[-1];
             value = stack_pointer[-2];
-            uint32_t type_version = read_u32(&next_instr[1].cache);
-            uint16_t hint = read_u16(&next_instr[3].cache);
+            uint32_t type_version = read_u32(&this_instr[2].cache);
+            uint16_t hint = read_u16(&this_instr[4].cache);
             PyTypeObject *tp = Py_TYPE(owner);
             assert(type_version != 0);
             DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
@@ -2710,17 +3030,19 @@
             dict->ma_version_tag = new_version;
             Py_DECREF(owner);
             STACK_SHRINK(2);
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(STORE_ATTR_SLOT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 5;
+            INSTRUCTION_STATS(STORE_ATTR_SLOT);
             PyObject *owner;
             PyObject *value;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
@@ -2728,7 +3050,7 @@
             // _STORE_ATTR_SLOT
             value = stack_pointer[-2];
             {
-                uint16_t index = read_u16(&next_instr[3].cache);
+                uint16_t index = read_u16(&this_instr[4].cache);
                 char *addr = (char *)owner + index;
                 STAT_INC(STORE_ATTR, hit);
                 PyObject *old_value = *(PyObject **)addr;
@@ -2737,12 +3059,15 @@
                 Py_DECREF(owner);
             }
             STACK_SHRINK(2);
-            next_instr += 4;
             DISPATCH();
         }
 
         TARGET(COMPARE_OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(COMPARE_OP);
             PREDICTED(COMPARE_OP);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
             PyObject *right;
             PyObject *left;
@@ -2750,14 +3075,13 @@
             right = stack_pointer[-1];
             left = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_CompareOp(left, right, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(COMPARE_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert((oparg >> 5) <= Py_GE);
             res = PyObject_RichCompare(left, right, oparg >> 5);
@@ -2772,11 +3096,13 @@
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(COMPARE_OP_FLOAT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(COMPARE_OP_FLOAT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -2795,11 +3121,13 @@
             // It's always a bool, so we don't care about oparg & 16.
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(COMPARE_OP_INT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(COMPARE_OP_INT);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -2822,11 +3150,13 @@
             // It's always a bool, so we don't care about oparg & 16.
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(COMPARE_OP_STR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(COMPARE_OP_STR);
             PyObject *right;
             PyObject *left;
             PyObject *res;
@@ -2846,11 +3176,13 @@
             // It's always a bool, so we don't care about oparg & 16.
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(IS_OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(IS_OP);
             PyObject *right;
             PyObject *left;
             PyObject *b;
@@ -2866,6 +3198,9 @@
         }
 
         TARGET(CONTAINS_OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CONTAINS_OP);
             PyObject *right;
             PyObject *left;
             PyObject *b;
@@ -2882,6 +3217,9 @@
         }
 
         TARGET(CHECK_EG_MATCH) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CHECK_EG_MATCH);
             PyObject *match_type;
             PyObject *exc_value;
             PyObject *rest;
@@ -2914,6 +3252,9 @@
         }
 
         TARGET(CHECK_EXC_MATCH) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CHECK_EXC_MATCH);
             PyObject *right;
             PyObject *left;
             PyObject *b;
@@ -2933,6 +3274,9 @@
         }
 
         TARGET(IMPORT_NAME) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(IMPORT_NAME);
             PyObject *fromlist;
             PyObject *level;
             PyObject *res;
@@ -2949,6 +3293,9 @@
         }
 
         TARGET(IMPORT_FROM) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(IMPORT_FROM);
             PyObject *from;
             PyObject *res;
             from = stack_pointer[-1];
@@ -2961,36 +3308,44 @@
         }
 
         TARGET(JUMP_FORWARD) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(JUMP_FORWARD);
             JUMPBY(oparg);
             DISPATCH();
         }
 
         TARGET(JUMP_BACKWARD) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(JUMP_BACKWARD);
             CHECK_EVAL_BREAKER();
-            _Py_CODEUNIT *here = next_instr - 1;
             assert(oparg <= INSTR_OFFSET());
             JUMPBY(1-oparg);
             #if ENABLE_SPECIALIZATION
-            here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER);
-            if (here[1].cache > tstate->interp->optimizer_backedge_threshold &&
+            this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER);
+            if (this_instr[1].cache > tstate->interp->optimizer_backedge_threshold &&
                 // Double-check that the opcode isn't instrumented or something:
-                here->op.code == JUMP_BACKWARD)
+                this_instr->op.code == JUMP_BACKWARD)
             {
                 OPT_STAT_INC(attempts);
-                int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer);
+                int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer);
                 if (optimized < 0) goto error;
                 if (optimized) {
                     // Rewind and enter the executor:
-                    assert(here->op.code == ENTER_EXECUTOR);
-                    next_instr = here;
+                    assert(this_instr->op.code == ENTER_EXECUTOR);
+                    next_instr = this_instr;
                 }
-                here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1);
+                this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1);
             }
             #endif  /* ENABLE_SPECIALIZATION */
             DISPATCH();
         }
 
         TARGET(ENTER_EXECUTOR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(ENTER_EXECUTOR);
             CHECK_EVAL_BREAKER();
 
             PyCodeObject *code = _PyFrame_GetCode(frame);
@@ -3009,34 +3364,41 @@
         }
 
         TARGET(POP_JUMP_IF_FALSE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
             PyObject *cond;
             cond = stack_pointer[-1];
             assert(PyBool_Check(cond));
             int flag = Py_IsFalse(cond);
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
             JUMPBY(oparg * flag);
             STACK_SHRINK(1);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(POP_JUMP_IF_TRUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
             PyObject *cond;
             cond = stack_pointer[-1];
             assert(PyBool_Check(cond));
             int flag = Py_IsTrue(cond);
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
             JUMPBY(oparg * flag);
             STACK_SHRINK(1);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(POP_JUMP_IF_NONE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(POP_JUMP_IF_NONE);
             PyObject *value;
             PyObject *b;
             PyObject *cond;
@@ -3057,16 +3419,18 @@
                 assert(PyBool_Check(cond));
                 int flag = Py_IsTrue(cond);
                 #if ENABLE_SPECIALIZATION
-                next_instr->cache = (next_instr->cache << 1) | flag;
+                this_instr[1].cache = (this_instr[1].cache << 1) | flag;
                 #endif
                 JUMPBY(oparg * flag);
             }
             STACK_SHRINK(1);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(POP_JUMP_IF_NOT_NONE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
             PyObject *value;
             PyObject *b;
             PyObject *cond;
@@ -3087,16 +3451,18 @@
                 assert(PyBool_Check(cond));
                 int flag = Py_IsFalse(cond);
                 #if ENABLE_SPECIALIZATION
-                next_instr->cache = (next_instr->cache << 1) | flag;
+                this_instr[1].cache = (this_instr[1].cache << 1) | flag;
                 #endif
                 JUMPBY(oparg * flag);
             }
             STACK_SHRINK(1);
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT);
             /* This bytecode is used in the `yield from` or `await` loop.
              * If there is an interrupt, we want it handled in the innermost
              * generator or coroutine, so we deliberately do not check it here.
@@ -3107,6 +3473,9 @@
         }
 
         TARGET(GET_LEN) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(GET_LEN);
             PyObject *obj;
             PyObject *len_o;
             obj = stack_pointer[-1];
@@ -3121,6 +3490,9 @@
         }
 
         TARGET(MATCH_CLASS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MATCH_CLASS);
             PyObject *names;
             PyObject *type;
             PyObject *subject;
@@ -3148,6 +3520,9 @@
         }
 
         TARGET(MATCH_MAPPING) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MATCH_MAPPING);
             PyObject *subject;
             PyObject *res;
             subject = stack_pointer[-1];
@@ -3159,6 +3534,9 @@
         }
 
         TARGET(MATCH_SEQUENCE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MATCH_SEQUENCE);
             PyObject *subject;
             PyObject *res;
             subject = stack_pointer[-1];
@@ -3170,6 +3548,9 @@
         }
 
         TARGET(MATCH_KEYS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MATCH_KEYS);
             PyObject *keys;
             PyObject *subject;
             PyObject *values_or_none;
@@ -3184,6 +3565,9 @@
         }
 
         TARGET(GET_ITER) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(GET_ITER);
             PyObject *iterable;
             PyObject *iter;
             iterable = stack_pointer[-1];
@@ -3196,6 +3580,9 @@
         }
 
         TARGET(GET_YIELD_FROM_ITER) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(GET_YIELD_FROM_ITER);
             PyObject *iterable;
             PyObject *iter;
             iterable = stack_pointer[-1];
@@ -3228,20 +3615,23 @@
         }
 
         TARGET(FOR_ITER) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(FOR_ITER);
             PREDICTED(FOR_ITER);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
             PyObject *iter;
             PyObject *next;
             iter = stack_pointer[-1];
             #if ENABLE_SPECIALIZATION
-            _PyForIterCache *cache = (_PyForIterCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_ForIter(iter, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(FOR_ITER, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
             next = (*Py_TYPE(iter)->tp_iternext)(iter);
@@ -3250,15 +3640,14 @@
                     if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
                         goto error;
                     }
-                    monitor_raise(tstate, frame, next_instr-1);
+                    monitor_raise(tstate, frame, this_instr);
                     _PyErr_Clear(tstate);
                 }
                 /* iterator ended normally */
-                assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR ||
-                       next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR);
+                assert(next_instr[oparg].op.code == END_FOR ||
+                       next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
                 Py_DECREF(iter);
                 STACK_SHRINK(1);
-                SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                 /* Jump forward oparg, then skip following END_FOR instruction */
                 JUMPBY(oparg + 1);
                 DISPATCH();
@@ -3266,40 +3655,44 @@
             // Common case: no jump, leave it to the code generator
             STACK_GROW(1);
             stack_pointer[-1] = next;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_FOR_ITER) {
-            _Py_CODEUNIT *here = frame->instr_ptr;
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER);
             _Py_CODEUNIT *target;
             PyObject *iter = TOP();
             PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
             if (next != NULL) {
                 PUSH(next);
-                target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER;
+                target = next_instr;
             }
             else {
                 if (_PyErr_Occurred(tstate)) {
                     if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
                         goto error;
                     }
-                    monitor_raise(tstate, frame, here);
+                    monitor_raise(tstate, frame, this_instr);
                     _PyErr_Clear(tstate);
                 }
                 /* iterator ended normally */
-                assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR ||
-                       next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR);
+                assert(next_instr[oparg].op.code == END_FOR ||
+                       next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
                 STACK_SHRINK(1);
                 Py_DECREF(iter);
                 /* Skip END_FOR */
-                target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1;
+                target = next_instr + oparg + 1;
             }
-            INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH);
+            INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH);
             DISPATCH();
         }
 
         TARGET(FOR_ITER_LIST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(FOR_ITER_LIST);
             PyObject *iter;
             PyObject *next;
             // _ITER_CHECK_LIST
@@ -3320,7 +3713,6 @@
                     }
                     Py_DECREF(iter);
                     STACK_SHRINK(1);
-                    SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                     /* Jump forward oparg, then skip following END_FOR instruction */
                     JUMPBY(oparg + 1);
                     DISPATCH();
@@ -3337,11 +3729,13 @@
             }
             STACK_GROW(1);
             stack_pointer[-1] = next;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(FOR_ITER_TUPLE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(FOR_ITER_TUPLE);
             PyObject *iter;
             PyObject *next;
             // _ITER_CHECK_TUPLE
@@ -3362,7 +3756,6 @@
                     }
                     Py_DECREF(iter);
                     STACK_SHRINK(1);
-                    SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                     /* Jump forward oparg, then skip following END_FOR instruction */
                     JUMPBY(oparg + 1);
                     DISPATCH();
@@ -3379,11 +3772,13 @@
             }
             STACK_GROW(1);
             stack_pointer[-1] = next;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(FOR_ITER_RANGE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(FOR_ITER_RANGE);
             PyObject *iter;
             PyObject *next;
             // _ITER_CHECK_RANGE
@@ -3400,7 +3795,6 @@
                 if (r->len <= 0) {
                     STACK_SHRINK(1);
                     Py_DECREF(r);
-                    SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
                     // Jump over END_FOR instruction.
                     JUMPBY(oparg + 1);
                     DISPATCH();
@@ -3419,11 +3813,13 @@
             }
             STACK_GROW(1);
             stack_pointer[-1] = next;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(FOR_ITER_GEN) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(FOR_ITER_GEN);
             PyObject *iter;
             iter = stack_pointer[-1];
             DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
@@ -3436,15 +3832,17 @@
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
-            SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
             assert(next_instr[oparg].op.code == END_FOR ||
                    next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
-            assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - this_instr);
             frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
             DISPATCH_INLINED(gen_frame);
         }
 
         TARGET(BEFORE_ASYNC_WITH) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BEFORE_ASYNC_WITH);
             PyObject *mgr;
             PyObject *exit;
             PyObject *res;
@@ -3485,6 +3883,9 @@
         }
 
         TARGET(BEFORE_WITH) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BEFORE_WITH);
             PyObject *mgr;
             PyObject *exit;
             PyObject *res;
@@ -3528,6 +3929,9 @@
         }
 
         TARGET(WITH_EXCEPT_START) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(WITH_EXCEPT_START);
             PyObject *val;
             PyObject *lasti;
             PyObject *exit_func;
@@ -3566,6 +3970,9 @@
         }
 
         TARGET(PUSH_EXC_INFO) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(PUSH_EXC_INFO);
             PyObject *new_exc;
             PyObject *prev_exc;
             new_exc = stack_pointer[-1];
@@ -3585,13 +3992,16 @@
         }
 
         TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
             PyObject *owner;
             PyObject *attr;
             PyObject *self;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -3604,14 +4014,14 @@
             }
             // _GUARD_KEYS_VERSION
             {
-                uint32_t keys_version = read_u32(&next_instr[3].cache);
+                uint32_t keys_version = read_u32(&this_instr[4].cache);
                 PyTypeObject *owner_cls = Py_TYPE(owner);
                 PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
                 DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
             }
             // _LOAD_ATTR_METHOD_WITH_VALUES
             {
-                PyObject *descr = read_obj(&next_instr[5].cache);
+                PyObject *descr = read_obj(&this_instr[6].cache);
                 assert(oparg & 1);
                 /* Cached method object */
                 STAT_INC(LOAD_ATTR, hit);
@@ -3623,25 +4033,27 @@
             STACK_GROW(1);
             stack_pointer[-2] = attr;
             stack_pointer[-1] = self;
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_METHOD_NO_DICT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
             PyObject *owner;
             PyObject *attr;
             PyObject *self;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
             }
             // _LOAD_ATTR_METHOD_NO_DICT
             {
-                PyObject *descr = read_obj(&next_instr[5].cache);
+                PyObject *descr = read_obj(&this_instr[6].cache);
                 assert(oparg & 1);
                 assert(Py_TYPE(owner)->tp_dictoffset == 0);
                 STAT_INC(LOAD_ATTR, hit);
@@ -3653,17 +4065,19 @@
             STACK_GROW(1);
             stack_pointer[-2] = attr;
             stack_pointer[-1] = self;
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
             PyObject *owner;
             PyObject *attr;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -3676,14 +4090,14 @@
             }
             // _GUARD_KEYS_VERSION
             {
-                uint32_t keys_version = read_u32(&next_instr[3].cache);
+                uint32_t keys_version = read_u32(&this_instr[4].cache);
                 PyTypeObject *owner_cls = Py_TYPE(owner);
                 PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
                 DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
             }
             // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
             {
-                PyObject *descr = read_obj(&next_instr[5].cache);
+                PyObject *descr = read_obj(&this_instr[6].cache);
                 assert((oparg & 1) == 0);
                 STAT_INC(LOAD_ATTR, hit);
                 assert(descr != NULL);
@@ -3691,24 +4105,26 @@
                 attr = Py_NewRef(descr);
             }
             stack_pointer[-1] = attr;
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
             PyObject *owner;
             PyObject *attr;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
             }
             // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
             {
-                PyObject *descr = read_obj(&next_instr[5].cache);
+                PyObject *descr = read_obj(&this_instr[6].cache);
                 assert((oparg & 1) == 0);
                 assert(Py_TYPE(owner)->tp_dictoffset == 0);
                 STAT_INC(LOAD_ATTR, hit);
@@ -3717,18 +4133,20 @@
                 attr = Py_NewRef(descr);
             }
             stack_pointer[-1] = attr;
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 10;
+            INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
             PyObject *owner;
             PyObject *attr;
             PyObject *self;
             // _GUARD_TYPE_VERSION
             owner = stack_pointer[-1];
             {
-                uint32_t type_version = read_u32(&next_instr[1].cache);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
                 PyTypeObject *tp = Py_TYPE(owner);
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -3743,7 +4161,7 @@
             }
             // _LOAD_ATTR_METHOD_LAZY_DICT
             {
-                PyObject *descr = read_obj(&next_instr[5].cache);
+                PyObject *descr = read_obj(&this_instr[6].cache);
                 assert(oparg & 1);
                 STAT_INC(LOAD_ATTR, hit);
                 assert(descr != NULL);
@@ -3754,11 +4172,13 @@
             STACK_GROW(1);
             stack_pointer[-2] = attr;
             stack_pointer[-1] = self;
-            next_instr += 9;
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_CALL) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(INSTRUMENTED_CALL);
             int is_meth = PEEK(oparg + 1) != NULL;
             int total_args = oparg + is_meth;
             PyObject *function = PEEK(oparg + 2);
@@ -3766,15 +4186,18 @@
                 &_PyInstrumentation_MISSING : PEEK(total_args);
             int err = _Py_call_instrumentation_2args(
                     tstate, PY_MONITORING_EVENT_CALL,
-                    frame, next_instr-1, function, arg);
+                    frame, this_instr, function, arg);
             if (err) goto error;
-            _PyCallCache *cache = (_PyCallCache *)next_instr;
-            INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+            INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             GO_TO_INSTRUCTION(CALL);
         }
 
         TARGET(CALL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL);
             PREDICTED(CALL);
+            _Py_CODEUNIT *this_instr = next_instr - 4;
             static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
             PyObject **args;
             PyObject *self_or_null;
@@ -3790,14 +4213,13 @@
                 total_args++;
             }
             #if ENABLE_SPECIALIZATION
-            _PyCallCache *cache = (_PyCallCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_Call(callable, next_instr, total_args);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(CALL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
                 args--;
@@ -3827,8 +4249,7 @@
                 if (new_frame == NULL) {
                     goto error;
                 }
-                SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
-                assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr);
+                assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
                 frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
                 DISPATCH_INLINED(new_frame);
             }
@@ -3843,12 +4264,12 @@
                 if (res == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                     if (err < 0) {
                         Py_CLEAR(res);
                     }
@@ -3863,12 +4284,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
             PyObject *null;
             PyObject *callable;
             PyObject *self;
@@ -3900,7 +4323,7 @@
             self_or_null = self;
             callable = func;
             {
-                uint32_t func_version = read_u32(&next_instr[1].cache);
+                uint32_t func_version = read_u32(&this_instr[2].cache);
                 DEOPT_IF(!PyFunction_Check(callable), CALL);
                 PyFunctionObject *func = (PyFunctionObject *)callable;
                 DEOPT_IF(func->func_version != func_version, CALL);
@@ -3930,10 +4353,9 @@
                 }
             }
             // _SAVE_RETURN_OFFSET
-            next_instr += 3;
             {
                 #if TIER_ONE
-                frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr);
+                frame->return_offset = (uint16_t)(next_instr - this_instr);
                 #endif
                 #if TIER_TWO
                 frame->return_offset = oparg;
@@ -3964,6 +4386,9 @@
         }
 
         TARGET(CALL_PY_EXACT_ARGS) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
             PyObject *self_or_null;
             PyObject *callable;
             PyObject **args;
@@ -3976,7 +4401,7 @@
             self_or_null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
             {
-                uint32_t func_version = read_u32(&next_instr[1].cache);
+                uint32_t func_version = read_u32(&this_instr[2].cache);
                 DEOPT_IF(!PyFunction_Check(callable), CALL);
                 PyFunctionObject *func = (PyFunctionObject *)callable;
                 DEOPT_IF(func->func_version != func_version, CALL);
@@ -4006,10 +4431,9 @@
                 }
             }
             // _SAVE_RETURN_OFFSET
-            next_instr += 3;
             {
                 #if TIER_ONE
-                frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr);
+                frame->return_offset = (uint16_t)(next_instr - this_instr);
                 #endif
                 #if TIER_TWO
                 frame->return_offset = oparg;
@@ -4040,13 +4464,16 @@
         }
 
         TARGET(CALL_PY_WITH_DEFAULTS) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
             args = stack_pointer - oparg;
             self_or_null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
-            uint32_t func_version = read_u32(&next_instr[1].cache);
+            uint32_t func_version = read_u32(&this_instr[2].cache);
             DEOPT_IF(tstate->interp->eval_frame, CALL);
             int argcount = oparg;
             if (self_or_null != NULL) {
@@ -4076,13 +4503,15 @@
             }
             // Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
             STACK_SHRINK(oparg + 2);
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
-            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
             DISPATCH_INLINED(new_frame);
         }
 
         TARGET(CALL_TYPE_1) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_TYPE_1);
             PyObject **args;
             PyObject *null;
             PyObject *callable;
@@ -4101,11 +4530,13 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(CALL_STR_1) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_STR_1);
             PyObject **args;
             PyObject *null;
             PyObject *callable;
@@ -4125,12 +4556,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_TUPLE_1) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_TUPLE_1);
             PyObject **args;
             PyObject *null;
             PyObject *callable;
@@ -4150,12 +4583,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_ALLOC_AND_ENTER_INIT) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
             PyObject **args;
             PyObject *null;
             PyObject *callable;
@@ -4167,7 +4602,7 @@
              * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
              * 3. Pushes the frame for ``__init__`` to the frame stack
              * */
-            _PyCallCache *cache = (_PyCallCache *)next_instr;
+            _PyCallCache *cache = (_PyCallCache *)&this_instr[1];
             DEOPT_IF(null != NULL, CALL);
             DEOPT_IF(!PyType_Check(callable), CALL);
             PyTypeObject *tp = (PyTypeObject *)callable;
@@ -4196,8 +4631,7 @@
             for (int i = 0; i < oparg; i++) {
                 init_frame->localsplus[i+1] = args[i];
             }
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
-            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr);
+            assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
             frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
             STACK_SHRINK(oparg+2);
             _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -4214,6 +4648,9 @@
         }
 
         TARGET(EXIT_INIT_CHECK) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(EXIT_INIT_CHECK);
             PyObject *should_be_none;
             should_be_none = stack_pointer[-1];
             assert(STACK_LEVEL() == 2);
@@ -4228,6 +4665,9 @@
         }
 
         TARGET(CALL_BUILTIN_CLASS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4254,12 +4694,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_BUILTIN_O) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_BUILTIN_O);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4294,12 +4736,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_BUILTIN_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_BUILTIN_FAST);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4338,12 +4782,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4376,12 +4822,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_LEN) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_LEN);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4413,11 +4861,13 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(CALL_ISINSTANCE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_ISINSTANCE);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4451,11 +4901,13 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             DISPATCH();
         }
 
         TARGET(CALL_LIST_APPEND) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_LIST_APPEND);
             PyObject **args;
             PyObject *self;
             PyObject *callable;
@@ -4474,13 +4926,16 @@
             Py_DECREF(self);
             Py_DECREF(callable);
             STACK_SHRINK(3);
-            // CALL + POP_TOP
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1);
-            assert(next_instr[-1].op.code == POP_TOP);
+            // Skip POP_TOP
+            assert(next_instr->op.code == POP_TOP);
+            SKIP_OVER(1);
             DISPATCH();
         }
 
         TARGET(CALL_METHOD_DESCRIPTOR_O) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4518,12 +4973,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4559,12 +5016,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4601,12 +5060,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(CALL_METHOD_DESCRIPTOR_FAST) {
+            frame->instr_ptr = next_instr;
+            next_instr += 4;
+            INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
             PyObject **args;
             PyObject *self_or_null;
             PyObject *callable;
@@ -4641,12 +5102,14 @@
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 3;
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_CALL_KW) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_CALL_KW);
             int is_meth = PEEK(oparg + 2) != NULL;
             int total_args = oparg + is_meth;
             PyObject *function = PEEK(oparg + 3);
@@ -4654,13 +5117,17 @@
                                             : PEEK(total_args + 1);
             int err = _Py_call_instrumentation_2args(
                     tstate, PY_MONITORING_EVENT_CALL,
-                    frame, next_instr - 1, function, arg);
+                    frame, this_instr, function, arg);
             if (err) goto error;
             GO_TO_INSTRUCTION(CALL_KW);
         }
 
         TARGET(CALL_KW) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CALL_KW);
             PREDICTED(CALL_KW);
+            _Py_CODEUNIT *this_instr = next_instr - 1;
             PyObject *kwnames;
             PyObject **args;
             PyObject *self_or_null;
@@ -4706,7 +5173,7 @@
                 if (new_frame == NULL) {
                     goto error;
                 }
-                assert(next_instr - frame->instr_ptr == 1);
+                assert(next_instr - this_instr == 1);
                 frame->return_offset = 1;
                 DISPATCH_INLINED(new_frame);
             }
@@ -4721,12 +5188,12 @@
                 if (res == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, callable, arg);
+                        frame, this_instr, callable, arg);
                     if (err < 0) {
                         Py_CLEAR(res);
                     }
@@ -4747,11 +5214,18 @@
         }
 
         TARGET(INSTRUMENTED_CALL_FUNCTION_EX) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
             GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
         }
 
         TARGET(CALL_FUNCTION_EX) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CALL_FUNCTION_EX);
             PREDICTED(CALL_FUNCTION_EX);
+            _Py_CODEUNIT *this_instr = next_instr - 1;
             PyObject *kwargs = NULL;
             PyObject *callargs;
             PyObject *func;
@@ -4781,18 +5255,18 @@
                     PyTuple_GET_ITEM(callargs, 0) : Py_None;
                 int err = _Py_call_instrumentation_2args(
                     tstate, PY_MONITORING_EVENT_CALL,
-                    frame, next_instr-1, func, arg);
+                    frame, this_instr, func, arg);
                 if (err) goto error;
                 result = PyObject_Call(func, callargs, kwargs);
                 if (result == NULL) {
                     _Py_call_instrumentation_exc2(
                         tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, next_instr-1, func, arg);
+                        frame, this_instr, func, arg);
                 }
                 else {
                     int err = _Py_call_instrumentation_2args(
                         tstate, PY_MONITORING_EVENT_C_RETURN,
-                        frame, next_instr-1, func, arg);
+                        frame, this_instr, func, arg);
                     if (err < 0) {
                         Py_CLEAR(result);
                     }
@@ -4815,7 +5289,7 @@
                     if (new_frame == NULL) {
                         goto error;
                     }
-                    assert(next_instr - frame->instr_ptr == 1);
+                    assert(next_instr - this_instr == 1);
                     frame->return_offset = 1;
                     DISPATCH_INLINED(new_frame);
                 }
@@ -4834,6 +5308,9 @@
         }
 
         TARGET(MAKE_FUNCTION) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(MAKE_FUNCTION);
             PyObject *codeobj;
             PyObject *func;
             codeobj = stack_pointer[-1];
@@ -4854,6 +5331,9 @@
         }
 
         TARGET(SET_FUNCTION_ATTRIBUTE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE);
             PyObject *func;
             PyObject *attr;
             func = stack_pointer[-1];
@@ -4888,6 +5368,9 @@
         }
 
         TARGET(RETURN_GENERATOR) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RETURN_GENERATOR);
             assert(PyFunction_Check(frame->f_funcobj));
             PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
             PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
@@ -4913,6 +5396,9 @@
         }
 
         TARGET(BUILD_SLICE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(BUILD_SLICE);
             PyObject *step = NULL;
             PyObject *stop;
             PyObject *start;
@@ -4932,6 +5418,9 @@
         }
 
         TARGET(CONVERT_VALUE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CONVERT_VALUE);
             PyObject *value;
             PyObject *result;
             value = stack_pointer[-1];
@@ -4946,6 +5435,9 @@
         }
 
         TARGET(FORMAT_SIMPLE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(FORMAT_SIMPLE);
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
@@ -4964,6 +5456,9 @@
         }
 
         TARGET(FORMAT_WITH_SPEC) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(FORMAT_WITH_SPEC);
             PyObject *fmt_spec;
             PyObject *value;
             PyObject *res;
@@ -4979,6 +5474,9 @@
         }
 
         TARGET(COPY) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(COPY);
             PyObject *bottom;
             PyObject *top;
             bottom = stack_pointer[-1 - (oparg-1)];
@@ -4990,7 +5488,11 @@
         }
 
         TARGET(BINARY_OP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(BINARY_OP);
             PREDICTED(BINARY_OP);
+            _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
             PyObject *rhs;
             PyObject *lhs;
@@ -4998,14 +5500,13 @@
             rhs = stack_pointer[-1];
             lhs = stack_pointer[-2];
             #if ENABLE_SPECIALIZATION
-            _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
-            if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
-                next_instr--;
+            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                next_instr = this_instr;
                 _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(BINARY_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
             assert(NB_ADD <= oparg);
             assert(oparg <= NB_INPLACE_XOR);
@@ -5016,11 +5517,13 @@
             if (res == NULL) goto pop_2_error;
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
-            next_instr += 1;
             DISPATCH();
         }
 
         TARGET(SWAP) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(SWAP);
             PyObject *top;
             PyObject *bottom;
             top = stack_pointer[-1];
@@ -5032,13 +5535,15 @@
         }
 
         TARGET(INSTRUMENTED_INSTRUCTION) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION);
             int next_opcode = _Py_call_instrumentation_instruction(
-                tstate, frame, next_instr-1);
+                tstate, frame, this_instr);
             if (next_opcode < 0) goto error;
-            next_instr--;
+            next_instr = this_instr;
             if (_PyOpcode_Caches[next_opcode]) {
-                _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
-                INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+                INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             }
             assert(next_opcode > 0 && next_opcode < 256);
             opcode = next_opcode;
@@ -5046,49 +5551,57 @@
         }
 
         TARGET(INSTRUMENTED_JUMP_FORWARD) {
-            _Py_CODEUNIT *here = frame->instr_ptr;
-            INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD);
+            INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_JUMP_BACKWARD) {
-            _Py_CODEUNIT *here = frame->instr_ptr;
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD);
             CHECK_EVAL_BREAKER();
-            INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP);
+            INSTRUMENTED_JUMP(this_instr, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP);
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE);
             PyObject *cond = POP();
             assert(PyBool_Check(cond));
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int flag = Py_IsTrue(cond);
             int offset = flag * oparg;
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            next_instr += 1;
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE);
             PyObject *cond = POP();
             assert(PyBool_Check(cond));
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int flag = Py_IsFalse(cond);
             int offset = flag * oparg;
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            next_instr += 1;
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE);
             PyObject *value = POP();
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int flag = Py_IsNone(value);
             int offset;
             if (flag) {
@@ -5099,16 +5612,17 @@
                 offset = 0;
             }
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | flag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            next_instr += 1;
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
+            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            next_instr += 2;
+            INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE);
             PyObject *value = POP();
-            _Py_CODEUNIT *here = frame->instr_ptr;
             int offset;
             int nflag = Py_IsNone(value);
             if (nflag) {
@@ -5119,14 +5633,16 @@
                 offset = oparg;
             }
             #if ENABLE_SPECIALIZATION
-            next_instr->cache = (next_instr->cache << 1) | !nflag;
+            this_instr[1].cache = (this_instr[1].cache << 1) | !nflag;
             #endif
-            INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            next_instr += 1;
+            INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
             DISPATCH();
         }
 
         TARGET(EXTENDED_ARG) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(EXTENDED_ARG);
             assert(oparg);
             opcode = next_instr->op.code;
             oparg = oparg << 8 | next_instr->op.arg;
@@ -5135,11 +5651,17 @@
         }
 
         TARGET(CACHE) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(CACHE);
             assert(0 && "Executing a cache.");
             Py_UNREACHABLE();
         }
 
         TARGET(RESERVED) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(RESERVED);
             assert(0 && "Executing RESERVED instruction.");
             Py_UNREACHABLE();
         }

diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py
index 53f715db6f24a5e9d84a55fd051ec7e0a43cef9f..180bd01302ec11264b1d02de6ac236bff2e0f9e4 100644 (file)
--- a/Tools/cases_generator/analysis.py
+++ b/Tools/cases_generator/analysis.py
@@ -432,7 +432,6 @@ class Analyzer:
             "LOAD_FAST_LOAD_FAST",
             "LOAD_CONST_LOAD_FAST",
             "STORE_FAST_STORE_FAST",
-            "_BINARY_OP_INPLACE_ADD_UNICODE",
             "POP_JUMP_IF_TRUE",
             "POP_JUMP_IF_FALSE",
             "_ITER_JUMP_LIST",

diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py
index c6b551675e3e7e9ee82dff1bf743a0d93ce3c55b..e9ed2a867ae2b3d6a236ac9dc54dad972e39fc5c 100644 (file)
--- a/Tools/cases_generator/instructions.py
+++ b/Tools/cases_generator/instructions.py
@@ -61,6 +61,7 @@ class Instruction:
     # Computed by constructor
     always_exits: str  # If the block always exits, its last line; else ""
     has_deopt: bool
+    needs_this_instr: bool
     cache_offset: int
     cache_effects: list[parsing.CacheEffect]
     input_effects: list[StackEffect]
@@ -87,6 +88,7 @@ class Instruction:
             effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect)
         ]
         self.cache_offset = sum(c.size for c in self.cache_effects)
+        self.needs_this_instr = variable_used(self.inst, "this_instr") or any(c.name != UNUSED for c in self.cache_effects)
         self.input_effects = [
             effect for effect in inst.inputs if isinstance(effect, StackEffect)
         ]
@@ -164,7 +166,8 @@ class Instruction:
                 func = f"read_u{bits}"
             if tier == TIER_ONE:
                 out.emit(
-                    f"{typ}{ceffect.name} = {func}(&next_instr[{active.offset}].cache);"
+                    f"{typ}{ceffect.name} = "
+                    f"{func}(&this_instr[{active.offset + 1}].cache);"
                 )
             else:
                 out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;")

diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py
index 69a6c10d6199c8a2d47bc34cfcc1db3579f7f9a7..123e38c524f49d3740eb0f0bef503c35ae99c348 100644 (file)
--- a/Tools/cases_generator/stacking.py
+++ b/Tools/cases_generator/stacking.py
@@ -365,8 +365,17 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None:
     ]
     out.emit("")
     with out.block(f"TARGET({mac.name})"):
+        needs_this = any(part.instr.needs_this_instr for part in parts)
+        if needs_this and not mac.predicted:
+            out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;")
+        else:
+            out.emit(f"frame->instr_ptr = next_instr;")
+        out.emit(f"next_instr += {mac.cache_offset+1};")
+        out.emit(f"INSTRUCTION_STATS({mac.name});")
         if mac.predicted:
             out.emit(f"PREDICTED({mac.name});")
+            if needs_this:
+                out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {mac.cache_offset+1};")
         out.static_assert_family_size(mac.name, mac.family, mac.cache_offset)
         try:
             next_instr_is_set = write_components(
@@ -375,8 +384,6 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None:
         except AssertionError as err:
             raise AssertionError(f"Error writing macro {mac.name}") from err
         if not parts[-1].instr.always_exits:
-            if not next_instr_is_set and mac.cache_offset:
-                out.emit(f"next_instr += {mac.cache_offset};")
             if parts[-1].instr.check_eval_breaker:
                 out.emit("CHECK_EVAL_BREAKER();")
             out.emit("DISPATCH();")
@@ -450,8 +457,6 @@ def write_components(
 
         if mgr.instr.name == "_SAVE_RETURN_OFFSET":
             next_instr_is_set = True
-            if cache_offset:
-                out.emit(f"next_instr += {cache_offset};")
             if tier == TIER_ONE:
                 assert_no_pokes(managers)