gh-106290: Fix edge cases around uops (#106319)

- Tweak uops debugging output
- Fix the bug from gh-106290
- Rename `SET_IP` to `SAVE_IP` (per https://github.com/faster-cpython/ideas/issues/558)
- Add a `SAVE_IP` uop at the start of the trace (ditto)
- Allow `unbound_local_error`; this gives us uops for `LOAD_FAST_CHECK`, `LOAD_CLOSURE`, and `DELETE_FAST`
- Longer traces
- Support `STORE_FAST_LOAD_FAST`, `STORE_FAST_STORE_FAST`
- Add deps on pycore_uops.h to Makefile(.pre.in)

diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h
index 0e88d7e7f4a3bdcb9b1d6337fd498c66bdbc7225..5ed275fb8576794eb0ff4e391b09947b358d9344 100644 (file)
--- a/Include/internal/pycore_uops.h
+++ b/Include/internal/pycore_uops.h
@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#define _Py_UOP_MAX_TRACE_LENGTH 16
+#define _Py_UOP_MAX_TRACE_LENGTH 32
 
 typedef struct {
     int opcode;

diff --git a/Makefile.pre.in b/Makefile.pre.in
index dcb3b5eb06a1e99c7f965724796bd9868e1116c7..41623bd2f1da7f763bc5c201e98818b98817d7c0 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1800,6 +1800,7 @@ PYTHON_HEADERS= \
                $(srcdir)/Include/internal/pycore_unionobject.h \
                $(srcdir)/Include/internal/pycore_unicodeobject.h \
                $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \
+               $(srcdir)/Include/internal/pycore_uops.h \
                $(srcdir)/Include/internal/pycore_warnings.h \
                $(srcdir)/Include/internal/pycore_weakref.h \
                $(DTRACE_HEADERS) \

diff --git a/Python/ceval.c b/Python/ceval.c
index 80ae85c24f0540b2e359065925b6ca4245a2c72a..6ce1a6a636ed718ada9291004489e0a028aed2c1 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -2773,24 +2773,26 @@ void Py_LeaveRecursiveCall(void)
 _PyInterpreterFrame *
 _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
 {
-#ifdef LLTRACE
+#ifdef Py_DEBUG
     char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
     int lltrace = 0;
     if (uop_debug != NULL && *uop_debug >= '0') {
         lltrace = *uop_debug - '0';  // TODO: Parse an int and all that
     }
-    if (lltrace >= 2) {
-        PyCodeObject *code = _PyFrame_GetCode(frame);
-        _Py_CODEUNIT *instr = frame->prev_instr + 1;
-        fprintf(stderr,
-                "Entering _PyUopExecute for %s (%s:%d) at offset %ld\n",
-                PyUnicode_AsUTF8(code->co_qualname),
-                PyUnicode_AsUTF8(code->co_filename),
-                code->co_firstlineno,
-                (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
-    }
+#define DPRINTF(level, ...) \
+    if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); }
+#else
+#define DPRINTF(level, ...)
 #endif
 
+    DPRINTF(3,
+            "Entering _PyUopExecute for %s (%s:%d) at offset %ld\n",
+            PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname),
+            PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename),
+            _PyFrame_GetCode(frame)->co_firstlineno,
+            (long)(frame->prev_instr + 1 -
+                   (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive));
+
     PyThreadState *tstate = _PyThreadState_GET();
     _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor;
 
@@ -2803,7 +2805,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
     }
 
     OBJECT_STAT_INC(optimization_traces_executed);
-    _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive - 1;
+    _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
     int pc = 0;
     int opcode;
     uint64_t operand;
@@ -2812,14 +2814,11 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
         opcode = self->trace[pc].opcode;
         operand = self->trace[pc].operand;
         oparg = (int)operand;
-#ifdef LLTRACE
-        if (lltrace >= 3) {
-            const char *opname = opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode];
-            int stack_level = (int)(stack_pointer - _PyFrame_Stackbase(frame));
-            fprintf(stderr, "  uop %s, operand %" PRIu64 ", stack_level %d\n",
-                    opname, operand, stack_level);
-        }
-#endif
+        DPRINTF(3,
+                "  uop %s, operand %" PRIu64 ", stack_level %d\n",
+                opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
+                operand,
+                (int)(stack_pointer - _PyFrame_Stackbase(frame)));
         pc++;
         OBJECT_STAT_INC(optimization_uops_executed);
         switch (opcode) {
@@ -2828,7 +2827,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
 #define ENABLE_SPECIALIZATION 0
 #include "executor_cases.c.h"
 
-            case SET_IP:
+            case SAVE_IP:
             {
                 frame->prev_instr = ip_offset + oparg;
                 break;
@@ -2836,6 +2835,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
 
             case EXIT_TRACE:
             {
+                frame->prev_instr--;  // Back up to just before destination
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 Py_DECREF(self);
                 return frame;
@@ -2850,6 +2850,13 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
         }
     }
 
+unbound_local_error:
+    format_exc_check_arg(tstate, PyExc_UnboundLocalError,
+        UNBOUNDLOCAL_ERROR_MSG,
+        PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+    );
+    goto error;
+
 pop_4_error:
     STACK_SHRINK(1);
 pop_3_error:
@@ -2861,11 +2868,7 @@ pop_1_error:
 error:
     // On ERROR_IF we return NULL as the frame.
     // The caller recovers the frame from cframe.current_frame.
-#ifdef LLTRACE
-    if (lltrace >= 2) {
-        fprintf(stderr, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
-    }
-#endif
+    DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
     _PyFrame_SetStackPointer(frame, stack_pointer);
     Py_DECREF(self);
     return NULL;
@@ -2873,11 +2876,8 @@ error:
 deoptimize:
     // On DEOPT_IF we just repeat the last instruction.
     // This presumes nothing was popped from the stack (nor pushed).
-#ifdef LLTRACE
-    if (lltrace >= 2) {
-        fprintf(stderr, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
-    }
-#endif
+    DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+    frame->prev_instr--;  // Back up to just before destination
     _PyFrame_SetStackPointer(frame, stack_pointer);
     Py_DECREF(self);
     return frame;

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 546b3d9f50ac76cb5c361e3f70107264b0bdd2ec..d1e0443db613d2760c6b68523ef14f5579403aa9 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -7,13 +7,25 @@
             break;
         }
 
+        case LOAD_FAST_CHECK: {
+            PyObject *value;
+            #line 182 "Python/bytecodes.c"
+            value = GETLOCAL(oparg);
+            if (value == NULL) goto unbound_local_error;
+            Py_INCREF(value);
+            #line 17 "Python/executor_cases.c.h"
+            STACK_GROW(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
         case LOAD_FAST: {
             PyObject *value;
             #line 188 "Python/bytecodes.c"
             value = GETLOCAL(oparg);
             assert(value != NULL);
             Py_INCREF(value);
-            #line 17 "Python/executor_cases.c.h"
+            #line 29 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = value;
             break;
@@ -25,7 +37,7 @@
             value = GETLOCAL(oparg);
             // do not use SETLOCAL here, it decrefs the old value
             GETLOCAL(oparg) = NULL;
-            #line 29 "Python/executor_cases.c.h"
+            #line 41 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = value;
             break;
@@ -36,7 +48,7 @@
             #line 209 "Python/bytecodes.c"
             value = GETITEM(FRAME_CO_CONSTS, oparg);
             Py_INCREF(value);
-            #line 40 "Python/executor_cases.c.h"
+            #line 52 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = value;
             break;
@@ -46,7 +58,7 @@
             PyObject *value = stack_pointer[-1];
             #line 214 "Python/bytecodes.c"
             SETLOCAL(oparg, value);
-            #line 50 "Python/executor_cases.c.h"
+            #line 62 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -54,7 +66,7 @@
         case POP_TOP: {
             PyObject *value = stack_pointer[-1];
             #line 237 "Python/bytecodes.c"
-            #line 58 "Python/executor_cases.c.h"
+            #line 70 "Python/executor_cases.c.h"
             Py_DECREF(value);
             STACK_SHRINK(1);
             break;
@@ -64,7 +76,7 @@
             PyObject *res;
             #line 241 "Python/bytecodes.c"
             res = NULL;
-            #line 68 "Python/executor_cases.c.h"
+            #line 80 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = res;
             break;
@@ -75,7 +87,7 @@
             PyObject *receiver = stack_pointer[-2];
             #line 260 "Python/bytecodes.c"
             Py_DECREF(receiver);
-            #line 79 "Python/executor_cases.c.h"
+            #line 91 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = value;
             break;
@@ -86,11 +98,11 @@
             PyObject *res;
             #line 275 "Python/bytecodes.c"
             res = PyNumber_Negative(value);
-            #line 90 "Python/executor_cases.c.h"
+            #line 102 "Python/executor_cases.c.h"
             Py_DECREF(value);
             #line 277 "Python/bytecodes.c"
             if (res == NULL) goto pop_1_error;
-            #line 94 "Python/executor_cases.c.h"
+            #line 106 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -101,7 +113,7 @@
             #line 281 "Python/bytecodes.c"
             assert(PyBool_Check(value));
             res = Py_IsFalse(value) ? Py_True : Py_False;
-            #line 105 "Python/executor_cases.c.h"
+            #line 117 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -111,7 +123,7 @@
             #line 313 "Python/bytecodes.c"
             DEOPT_IF(!PyBool_Check(value), TO_BOOL);
             STAT_INC(TO_BOOL, hit);
-            #line 115 "Python/executor_cases.c.h"
+            #line 127 "Python/executor_cases.c.h"
             break;
         }
 
@@ -126,12 +138,12 @@
                 res = Py_False;
             }
             else {
-            #line 130 "Python/executor_cases.c.h"
+            #line 142 "Python/executor_cases.c.h"
                 Py_DECREF(value);
             #line 326 "Python/bytecodes.c"
                 res = Py_True;
             }
-            #line 135 "Python/executor_cases.c.h"
+            #line 147 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -143,7 +155,7 @@
             DEOPT_IF(!PyList_CheckExact(value), TO_BOOL);
             STAT_INC(TO_BOOL, hit);
             res = Py_SIZE(value) ? Py_True : Py_False;
-            #line 147 "Python/executor_cases.c.h"
+            #line 159 "Python/executor_cases.c.h"
             Py_DECREF(value);
             stack_pointer[-1] = res;
             break;
@@ -157,7 +169,7 @@
             DEOPT_IF(!Py_IsNone(value), TO_BOOL);
             STAT_INC(TO_BOOL, hit);
             res = Py_False;
-            #line 161 "Python/executor_cases.c.h"
+            #line 173 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -174,12 +186,12 @@
             }
             else {
                 assert(Py_SIZE(value));
-            #line 178 "Python/executor_cases.c.h"
+            #line 190 "Python/executor_cases.c.h"
                 Py_DECREF(value);
             #line 354 "Python/bytecodes.c"
                 res = Py_True;
             }
-            #line 183 "Python/executor_cases.c.h"
+            #line 195 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -193,11 +205,11 @@
             assert(version);
             DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
             STAT_INC(TO_BOOL, hit);
-            #line 197 "Python/executor_cases.c.h"
+            #line 209 "Python/executor_cases.c.h"
             Py_DECREF(value);
             #line 364 "Python/bytecodes.c"
             res = Py_True;
-            #line 201 "Python/executor_cases.c.h"
+            #line 213 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -207,11 +219,11 @@
             PyObject *res;
             #line 368 "Python/bytecodes.c"
             res = PyNumber_Invert(value);
-            #line 211 "Python/executor_cases.c.h"
+            #line 223 "Python/executor_cases.c.h"
             Py_DECREF(value);
             #line 370 "Python/bytecodes.c"
             if (res == NULL) goto pop_1_error;
-            #line 215 "Python/executor_cases.c.h"
+            #line 227 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -222,7 +234,7 @@
             #line 386 "Python/bytecodes.c"
             DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
             DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
-            #line 226 "Python/executor_cases.c.h"
+            #line 238 "Python/executor_cases.c.h"
             break;
         }
 
@@ -236,7 +248,7 @@
             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
             if (res == NULL) goto pop_2_error;
-            #line 240 "Python/executor_cases.c.h"
+            #line 252 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -252,7 +264,7 @@
             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
             if (res == NULL) goto pop_2_error;
-            #line 256 "Python/executor_cases.c.h"
+            #line 268 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -268,7 +280,7 @@
             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
             if (res == NULL) goto pop_2_error;
-            #line 272 "Python/executor_cases.c.h"
+            #line 284 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -280,7 +292,7 @@
             #line 422 "Python/bytecodes.c"
             DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
             DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
-            #line 284 "Python/executor_cases.c.h"
+            #line 296 "Python/executor_cases.c.h"
             break;
         }
 
@@ -294,7 +306,7 @@
                 ((PyFloatObject *)left)->ob_fval *
                 ((PyFloatObject *)right)->ob_fval;
             DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
-            #line 298 "Python/executor_cases.c.h"
+            #line 310 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -310,7 +322,7 @@
                 ((PyFloatObject *)left)->ob_fval +
                 ((PyFloatObject *)right)->ob_fval;
             DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
-            #line 314 "Python/executor_cases.c.h"
+            #line 326 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -326,7 +338,7 @@
                 ((PyFloatObject *)left)->ob_fval -
                 ((PyFloatObject *)right)->ob_fval;
             DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
-            #line 330 "Python/executor_cases.c.h"
+            #line 342 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -338,7 +350,7 @@
             #line 458 "Python/bytecodes.c"
             DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
             DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
-            #line 342 "Python/executor_cases.c.h"
+            #line 354 "Python/executor_cases.c.h"
             break;
         }
 
@@ -352,7 +364,7 @@
             _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
             _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
             if (res == NULL) goto pop_2_error;
-            #line 356 "Python/executor_cases.c.h"
+            #line 368 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -376,7 +388,7 @@
             }
             Py_DECREF(container);
             if (res == NULL) goto pop_3_error;
-            #line 380 "Python/executor_cases.c.h"
+            #line 392 "Python/executor_cases.c.h"
             STACK_SHRINK(2);
             stack_pointer[-1] = res;
             break;
@@ -400,7 +412,7 @@
             Py_DECREF(v);
             Py_DECREF(container);
             if (err) goto pop_4_error;
-            #line 404 "Python/executor_cases.c.h"
+            #line 416 "Python/executor_cases.c.h"
             STACK_SHRINK(4);
             break;
         }
@@ -423,7 +435,7 @@
             Py_INCREF(res);
             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
             Py_DECREF(list);
-            #line 427 "Python/executor_cases.c.h"
+            #line 439 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -447,7 +459,7 @@
             Py_INCREF(res);
             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
             Py_DECREF(tuple);
-            #line 451 "Python/executor_cases.c.h"
+            #line 463 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -465,14 +477,14 @@
                 if (!_PyErr_Occurred(tstate)) {
                     _PyErr_SetKeyError(sub);
                 }
-            #line 469 "Python/executor_cases.c.h"
+            #line 481 "Python/executor_cases.c.h"
                 Py_DECREF(dict);
                 Py_DECREF(sub);
             #line 603 "Python/bytecodes.c"
                 if (true) goto pop_2_error;
             }
             Py_INCREF(res);  // Do this before DECREF'ing dict, sub
-            #line 476 "Python/executor_cases.c.h"
+            #line 488 "Python/executor_cases.c.h"
             Py_DECREF(dict);
             Py_DECREF(sub);
             STACK_SHRINK(1);
@@ -485,7 +497,7 @@
             PyObject *list = stack_pointer[-(2 + (oparg-1))];
             #line 635 "Python/bytecodes.c"
             if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
-            #line 489 "Python/executor_cases.c.h"
+            #line 501 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -495,11 +507,11 @@
             PyObject *set = stack_pointer[-(2 + (oparg-1))];
             #line 639 "Python/bytecodes.c"
             int err = PySet_Add(set, v);
-            #line 499 "Python/executor_cases.c.h"
+            #line 511 "Python/executor_cases.c.h"
             Py_DECREF(v);
             #line 641 "Python/bytecodes.c"
             if (err) goto pop_1_error;
-            #line 503 "Python/executor_cases.c.h"
+            #line 515 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -525,7 +537,7 @@
             Py_DECREF(old_value);
             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
             Py_DECREF(list);
-            #line 529 "Python/executor_cases.c.h"
+            #line 541 "Python/executor_cases.c.h"
             STACK_SHRINK(3);
             break;
         }
@@ -540,7 +552,7 @@
             int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
             Py_DECREF(dict);
             if (err) goto pop_3_error;
-            #line 544 "Python/executor_cases.c.h"
+            #line 556 "Python/executor_cases.c.h"
             STACK_SHRINK(3);
             break;
         }
@@ -551,12 +563,12 @@
             #line 697 "Python/bytecodes.c"
             /* del container[sub] */
             int err = PyObject_DelItem(container, sub);
-            #line 555 "Python/executor_cases.c.h"
+            #line 567 "Python/executor_cases.c.h"
             Py_DECREF(container);
             Py_DECREF(sub);
             #line 700 "Python/bytecodes.c"
             if (err) goto pop_2_error;
-            #line 560 "Python/executor_cases.c.h"
+            #line 572 "Python/executor_cases.c.h"
             STACK_SHRINK(2);
             break;
         }
@@ -567,11 +579,11 @@
             #line 704 "Python/bytecodes.c"
             assert(oparg <= MAX_INTRINSIC_1);
             res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value);
-            #line 571 "Python/executor_cases.c.h"
+            #line 583 "Python/executor_cases.c.h"
             Py_DECREF(value);
             #line 707 "Python/bytecodes.c"
             if (res == NULL) goto pop_1_error;
-            #line 575 "Python/executor_cases.c.h"
+            #line 587 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -583,12 +595,12 @@
             #line 711 "Python/bytecodes.c"
             assert(oparg <= MAX_INTRINSIC_2);
             res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1);
-            #line 587 "Python/executor_cases.c.h"
+            #line 599 "Python/executor_cases.c.h"
             Py_DECREF(value2);
             Py_DECREF(value1);
             #line 714 "Python/bytecodes.c"
             if (res == NULL) goto pop_2_error;
-            #line 592 "Python/executor_cases.c.h"
+            #line 604 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -610,14 +622,14 @@
                               "'async for' requires an object with "
                               "__aiter__ method, got %.100s",
                               type->tp_name);
-            #line 614 "Python/executor_cases.c.h"
+            #line 626 "Python/executor_cases.c.h"
                 Py_DECREF(obj);
             #line 832 "Python/bytecodes.c"
                 if (true) goto pop_1_error;
             }
 
             iter = (*getter)(obj);
-            #line 621 "Python/executor_cases.c.h"
+            #line 633 "Python/executor_cases.c.h"
             Py_DECREF(obj);
             #line 837 "Python/bytecodes.c"
             if (iter == NULL) goto pop_1_error;
@@ -632,7 +644,7 @@
                 Py_DECREF(iter);
                 if (true) goto pop_1_error;
             }
-            #line 636 "Python/executor_cases.c.h"
+            #line 648 "Python/executor_cases.c.h"
             stack_pointer[-1] = iter;
             break;
         }
@@ -683,7 +695,7 @@
                     Py_DECREF(next_iter);
                 }
             }
-            #line 687 "Python/executor_cases.c.h"
+            #line 699 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = awaitable;
             break;
@@ -699,7 +711,7 @@
                 format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
             }
 
-            #line 703 "Python/executor_cases.c.h"
+            #line 715 "Python/executor_cases.c.h"
             Py_DECREF(iterable);
             #line 904 "Python/bytecodes.c"
 
@@ -718,7 +730,7 @@
             }
 
             if (iter == NULL) goto pop_1_error;
-            #line 722 "Python/executor_cases.c.h"
+            #line 734 "Python/executor_cases.c.h"
             stack_pointer[-1] = iter;
             break;
         }
@@ -728,7 +740,7 @@
             #line 1034 "Python/bytecodes.c"
             _PyErr_StackItem *exc_info = tstate->exc_info;
             Py_XSETREF(exc_info->exc_value, exc_value);
-            #line 732 "Python/executor_cases.c.h"
+            #line 744 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -737,7 +749,7 @@
             PyObject *value;
             #line 1085 "Python/bytecodes.c"
             value = Py_NewRef(PyExc_AssertionError);
-            #line 741 "Python/executor_cases.c.h"
+            #line 753 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = value;
             break;
@@ -767,7 +779,7 @@
                     if (true) goto error;
                 }
             }
-            #line 771 "Python/executor_cases.c.h"
+            #line 783 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = bc;
             break;
@@ -782,7 +794,7 @@
             if (ns == NULL) {
                 _PyErr_Format(tstate, PyExc_SystemError,
                               "no locals found when storing %R", name);
-            #line 786 "Python/executor_cases.c.h"
+            #line 798 "Python/executor_cases.c.h"
                 Py_DECREF(v);
             #line 1121 "Python/bytecodes.c"
                 if (true) goto pop_1_error;
@@ -791,11 +803,11 @@
                 err = PyDict_SetItem(ns, name, v);
             else
                 err = PyObject_SetItem(ns, name, v);
-            #line 795 "Python/executor_cases.c.h"
+            #line 807 "Python/executor_cases.c.h"
             Py_DECREF(v);
             #line 1128 "Python/bytecodes.c"
             if (err) goto pop_1_error;
-            #line 799 "Python/executor_cases.c.h"
+            #line 811 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -818,7 +830,7 @@
                                      name);
                 goto error;
             }
-            #line 822 "Python/executor_cases.c.h"
+            #line 834 "Python/executor_cases.c.h"
             break;
         }
 
@@ -832,7 +844,7 @@
             STAT_INC(UNPACK_SEQUENCE, hit);
             values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
             values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
-            #line 836 "Python/executor_cases.c.h"
+            #line 848 "Python/executor_cases.c.h"
             Py_DECREF(seq);
             STACK_SHRINK(1);
             STACK_GROW(oparg);
@@ -850,7 +862,7 @@
             for (int i = oparg; --i >= 0; ) {
                 *values++ = Py_NewRef(items[i]);
             }
-            #line 854 "Python/executor_cases.c.h"
+            #line 866 "Python/executor_cases.c.h"
             Py_DECREF(seq);
             STACK_SHRINK(1);
             STACK_GROW(oparg);
@@ -868,7 +880,7 @@
             for (int i = oparg; --i >= 0; ) {
                 *values++ = Py_NewRef(items[i]);
             }
-            #line 872 "Python/executor_cases.c.h"
+            #line 884 "Python/executor_cases.c.h"
             Py_DECREF(seq);
             STACK_SHRINK(1);
             STACK_GROW(oparg);
@@ -881,11 +893,11 @@
             int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
             PyObject **top = stack_pointer + totalargs - 1;
             int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
-            #line 885 "Python/executor_cases.c.h"
+            #line 897 "Python/executor_cases.c.h"
             Py_DECREF(seq);
             #line 1211 "Python/bytecodes.c"
             if (res == 0) goto pop_1_error;
-            #line 889 "Python/executor_cases.c.h"
+            #line 901 "Python/executor_cases.c.h"
             STACK_GROW((oparg & 0xFF) + (oparg >> 8));
             break;
         }
@@ -895,11 +907,11 @@
             #line 1242 "Python/bytecodes.c"
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
-            #line 899 "Python/executor_cases.c.h"
+            #line 911 "Python/executor_cases.c.h"
             Py_DECREF(owner);
             #line 1245 "Python/bytecodes.c"
             if (err) goto pop_1_error;
-            #line 903 "Python/executor_cases.c.h"
+            #line 915 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -909,11 +921,11 @@
             #line 1249 "Python/bytecodes.c"
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyDict_SetItem(GLOBALS(), name, v);
-            #line 913 "Python/executor_cases.c.h"
+            #line 925 "Python/executor_cases.c.h"
             Py_DECREF(v);
             #line 1252 "Python/bytecodes.c"
             if (err) goto pop_1_error;
-            #line 917 "Python/executor_cases.c.h"
+            #line 929 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -931,7 +943,7 @@
                 }
                 goto error;
             }
-            #line 935 "Python/executor_cases.c.h"
+            #line 947 "Python/executor_cases.c.h"
             break;
         }
 
@@ -945,7 +957,7 @@
                 if (true) goto error;
             }
             Py_INCREF(locals);
-            #line 949 "Python/executor_cases.c.h"
+            #line 961 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = locals;
             break;
@@ -1011,11 +1023,20 @@
                     }
                 }
             }
-            #line 1015 "Python/executor_cases.c.h"
+            #line 1027 "Python/executor_cases.c.h"
             stack_pointer[-1] = v;
             break;
         }
 
+        case DELETE_FAST: {
+            #line 1435 "Python/bytecodes.c"
+            PyObject *v = GETLOCAL(oparg);
+            if (v == NULL) goto unbound_local_error;
+            SETLOCAL(oparg, NULL);
+            #line 1037 "Python/executor_cases.c.h"
+            break;
+        }
+
         case DELETE_DEREF: {
             #line 1452 "Python/bytecodes.c"
             PyObject *cell = GETLOCAL(oparg);
@@ -1028,7 +1049,7 @@
             }
             PyCell_SET(cell, NULL);
             Py_DECREF(oldobj);
-            #line 1032 "Python/executor_cases.c.h"
+            #line 1053 "Python/executor_cases.c.h"
             break;
         }
 
@@ -1070,7 +1091,7 @@
                 }
                 Py_INCREF(value);
             }
-            #line 1074 "Python/executor_cases.c.h"
+            #line 1095 "Python/executor_cases.c.h"
             stack_pointer[-1] = value;
             break;
         }
@@ -1085,7 +1106,7 @@
                 if (true) goto error;
             }
             Py_INCREF(value);
-            #line 1089 "Python/executor_cases.c.h"
+            #line 1110 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = value;
             break;
@@ -1098,7 +1119,7 @@
             PyObject *oldobj = PyCell_GET(cell);
             PyCell_SET(cell, v);
             Py_XDECREF(oldobj);
-            #line 1102 "Python/executor_cases.c.h"
+            #line 1123 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -1115,7 +1136,7 @@
                 PyObject *o = PyTuple_GET_ITEM(closure, i);
                 frame->localsplus[offset + i] = Py_NewRef(o);
             }
-            #line 1119 "Python/executor_cases.c.h"
+            #line 1140 "Python/executor_cases.c.h"
             break;
         }
 
@@ -1124,13 +1145,13 @@
             PyObject *str;
             #line 1532 "Python/bytecodes.c"
             str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
-            #line 1128 "Python/executor_cases.c.h"
+            #line 1149 "Python/executor_cases.c.h"
             for (int _i = oparg; --_i >= 0;) {
                 Py_DECREF(pieces[_i]);
             }
             #line 1534 "Python/bytecodes.c"
             if (str == NULL) { STACK_SHRINK(oparg); goto error; }
-            #line 1134 "Python/executor_cases.c.h"
+            #line 1155 "Python/executor_cases.c.h"
             STACK_SHRINK(oparg);
             STACK_GROW(1);
             stack_pointer[-1] = str;
@@ -1143,7 +1164,7 @@
             #line 1538 "Python/bytecodes.c"
             tup = _PyTuple_FromArraySteal(values, oparg);
             if (tup == NULL) { STACK_SHRINK(oparg); goto error; }
-            #line 1147 "Python/executor_cases.c.h"
+            #line 1168 "Python/executor_cases.c.h"
             STACK_SHRINK(oparg);
             STACK_GROW(1);
             stack_pointer[-1] = tup;
@@ -1156,7 +1177,7 @@
             #line 1543 "Python/bytecodes.c"
             list = _PyList_FromArraySteal(values, oparg);
             if (list == NULL) { STACK_SHRINK(oparg); goto error; }
-            #line 1160 "Python/executor_cases.c.h"
+            #line 1181 "Python/executor_cases.c.h"
             STACK_SHRINK(oparg);
             STACK_GROW(1);
             stack_pointer[-1] = list;
@@ -1177,13 +1198,13 @@
                           "Value after * must be an iterable, not %.200s",
                           Py_TYPE(iterable)->tp_name);
                 }
-            #line 1181 "Python/executor_cases.c.h"
+            #line 1202 "Python/executor_cases.c.h"
                 Py_DECREF(iterable);
             #line 1559 "Python/bytecodes.c"
                 if (true) goto pop_1_error;
             }
             assert(Py_IsNone(none_val));
-            #line 1187 "Python/executor_cases.c.h"
+            #line 1208 "Python/executor_cases.c.h"
             Py_DECREF(iterable);
             STACK_SHRINK(1);
             break;
@@ -1194,11 +1215,11 @@
             PyObject *set = stack_pointer[-(2 + (oparg-1))];
             #line 1566 "Python/bytecodes.c"
             int err = _PySet_Update(set, iterable);
-            #line 1198 "Python/executor_cases.c.h"
+            #line 1219 "Python/executor_cases.c.h"
             Py_DECREF(iterable);
             #line 1568 "Python/bytecodes.c"
             if (err < 0) goto pop_1_error;
-            #line 1202 "Python/executor_cases.c.h"
+            #line 1223 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -1221,7 +1242,7 @@
                 Py_DECREF(set);
                 if (true) { STACK_SHRINK(oparg); goto error; }
             }
-            #line 1225 "Python/executor_cases.c.h"
+            #line 1246 "Python/executor_cases.c.h"
             STACK_SHRINK(oparg);
             STACK_GROW(1);
             stack_pointer[-1] = set;
@@ -1239,13 +1260,13 @@
             if (map == NULL)
                 goto error;
 
-            #line 1243 "Python/executor_cases.c.h"
+            #line 1264 "Python/executor_cases.c.h"
             for (int _i = oparg*2; --_i >= 0;) {
                 Py_DECREF(values[_i]);
             }
             #line 1597 "Python/bytecodes.c"
             if (map == NULL) { STACK_SHRINK(oparg*2); goto error; }
-            #line 1249 "Python/executor_cases.c.h"
+            #line 1270 "Python/executor_cases.c.h"
             STACK_SHRINK(oparg*2);
             STACK_GROW(1);
             stack_pointer[-1] = map;
@@ -1293,7 +1314,7 @@
                     Py_DECREF(ann_dict);
                 }
             }
-            #line 1297 "Python/executor_cases.c.h"
+            #line 1318 "Python/executor_cases.c.h"
             break;
         }
 
@@ -1311,14 +1332,14 @@
             map = _PyDict_FromItems(
                     &PyTuple_GET_ITEM(keys, 0), 1,
                     values, 1, oparg);
-            #line 1315 "Python/executor_cases.c.h"
+            #line 1336 "Python/executor_cases.c.h"
             for (int _i = oparg; --_i >= 0;) {
                 Py_DECREF(values[_i]);
             }
             Py_DECREF(keys);
             #line 1653 "Python/bytecodes.c"
             if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; }
-            #line 1322 "Python/executor_cases.c.h"
+            #line 1343 "Python/executor_cases.c.h"
             STACK_SHRINK(oparg);
             stack_pointer[-1] = map;
             break;
@@ -1334,12 +1355,12 @@
                                     "'%.200s' object is not a mapping",
                                     Py_TYPE(update)->tp_name);
                 }
-            #line 1338 "Python/executor_cases.c.h"
+            #line 1359 "Python/executor_cases.c.h"
                 Py_DECREF(update);
             #line 1665 "Python/bytecodes.c"
                 if (true) goto pop_1_error;
             }
-            #line 1343 "Python/executor_cases.c.h"
+            #line 1364 "Python/executor_cases.c.h"
             Py_DECREF(update);
             STACK_SHRINK(1);
             break;
@@ -1352,12 +1373,12 @@
 
             if (_PyDict_MergeEx(dict, update, 2) < 0) {
                 format_kwargs_error(tstate, PEEK(3 + oparg), update);
-            #line 1356 "Python/executor_cases.c.h"
+            #line 1377 "Python/executor_cases.c.h"
                 Py_DECREF(update);
             #line 1676 "Python/bytecodes.c"
                 if (true) goto pop_1_error;
             }
-            #line 1361 "Python/executor_cases.c.h"
+            #line 1382 "Python/executor_cases.c.h"
             Py_DECREF(update);
             STACK_SHRINK(1);
             break;
@@ -1372,7 +1393,7 @@
             /* dict[key] = value */
             // Do not DECREF INPUTS because the function steals the references
             if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
-            #line 1376 "Python/executor_cases.c.h"
+            #line 1397 "Python/executor_cases.c.h"
             STACK_SHRINK(2);
             break;
         }
@@ -1390,13 +1411,13 @@
             STAT_INC(LOAD_SUPER_ATTR, hit);
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
             res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
-            #line 1394 "Python/executor_cases.c.h"
+            #line 1415 "Python/executor_cases.c.h"
             Py_DECREF(global_super);
             Py_DECREF(class);
             Py_DECREF(self);
             #line 1772 "Python/bytecodes.c"
             if (res == NULL) goto pop_3_error;
-            #line 1400 "Python/executor_cases.c.h"
+            #line 1421 "Python/executor_cases.c.h"
             STACK_SHRINK(2);
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1] = res;
@@ -1433,7 +1454,7 @@
                 res = res2;
                 res2 = NULL;
             }
-            #line 1437 "Python/executor_cases.c.h"
+            #line 1458 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             stack_pointer[-2] = res2;
@@ -1456,7 +1477,7 @@
             _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
             res = (sign_ish & oparg) ? Py_True : Py_False;
             // It's always a bool, so we don't care about oparg & 16.
-            #line 1460 "Python/executor_cases.c.h"
+            #line 1481 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -1482,7 +1503,7 @@
             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
             res = (sign_ish & oparg) ? Py_True : Py_False;
             // It's always a bool, so we don't care about oparg & 16.
-            #line 1486 "Python/executor_cases.c.h"
+            #line 1507 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -1505,7 +1526,7 @@
             assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
             res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
             // It's always a bool, so we don't care about oparg & 16.
-            #line 1509 "Python/executor_cases.c.h"
+            #line 1530 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -1517,12 +1538,12 @@
             PyObject *b;
             #line 2163 "Python/bytecodes.c"
             int res = Py_Is(left, right) ^ oparg;
-            #line 1521 "Python/executor_cases.c.h"
+            #line 1542 "Python/executor_cases.c.h"
             Py_DECREF(left);
             Py_DECREF(right);
             #line 2165 "Python/bytecodes.c"
             b = res ? Py_True : Py_False;
-            #line 1526 "Python/executor_cases.c.h"
+            #line 1547 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = b;
             break;
@@ -1534,13 +1555,13 @@
             PyObject *b;
             #line 2169 "Python/bytecodes.c"
             int res = PySequence_Contains(right, left);
-            #line 1538 "Python/executor_cases.c.h"
+            #line 1559 "Python/executor_cases.c.h"
             Py_DECREF(left);
             Py_DECREF(right);
             #line 2171 "Python/bytecodes.c"
             if (res < 0) goto pop_2_error;
             b = (res ^ oparg) ? Py_True : Py_False;
-            #line 1544 "Python/executor_cases.c.h"
+            #line 1565 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = b;
             break;
@@ -1553,7 +1574,7 @@
             PyObject *match;
             #line 2176 "Python/bytecodes.c"
             if (check_except_star_type_valid(tstate, match_type) < 0) {
-            #line 1557 "Python/executor_cases.c.h"
+            #line 1578 "Python/executor_cases.c.h"
                 Py_DECREF(exc_value);
                 Py_DECREF(match_type);
             #line 2178 "Python/bytecodes.c"
@@ -1564,7 +1585,7 @@
             rest = NULL;
             int res = exception_group_match(exc_value, match_type,
                                             &match, &rest);
-            #line 1568 "Python/executor_cases.c.h"
+            #line 1589 "Python/executor_cases.c.h"
             Py_DECREF(exc_value);
             Py_DECREF(match_type);
             #line 2186 "Python/bytecodes.c"
@@ -1576,7 +1597,7 @@
             if (!Py_IsNone(match)) {
                 PyErr_SetHandledException(match);
             }
-            #line 1580 "Python/executor_cases.c.h"
+            #line 1601 "Python/executor_cases.c.h"
             stack_pointer[-1] = match;
             stack_pointer[-2] = rest;
             break;
@@ -1589,18 +1610,18 @@
             #line 2197 "Python/bytecodes.c"
             assert(PyExceptionInstance_Check(left));
             if (check_except_type_valid(tstate, right) < 0) {
-            #line 1593 "Python/executor_cases.c.h"
+            #line 1614 "Python/executor_cases.c.h"
                  Py_DECREF(right);
             #line 2200 "Python/bytecodes.c"
                  if (true) goto pop_1_error;
             }
 
             int res = PyErr_GivenExceptionMatches(left, right);
-            #line 1600 "Python/executor_cases.c.h"
+            #line 1621 "Python/executor_cases.c.h"
             Py_DECREF(right);
             #line 2205 "Python/bytecodes.c"
             b = res ? Py_True : Py_False;
-            #line 1604 "Python/executor_cases.c.h"
+            #line 1625 "Python/executor_cases.c.h"
             stack_pointer[-1] = b;
             break;
         }
@@ -1614,7 +1635,7 @@
             if (len_i < 0) goto error;
             len_o = PyLong_FromSsize_t(len_i);
             if (len_o == NULL) goto error;
-            #line 1618 "Python/executor_cases.c.h"
+            #line 1639 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = len_o;
             break;
@@ -1630,7 +1651,7 @@
             // None on failure.
             assert(PyTuple_CheckExact(names));
             attrs = match_class(tstate, subject, type, oparg, names);
-            #line 1634 "Python/executor_cases.c.h"
+            #line 1655 "Python/executor_cases.c.h"
             Py_DECREF(subject);
             Py_DECREF(type);
             Py_DECREF(names);
@@ -1642,7 +1663,7 @@
                 if (_PyErr_Occurred(tstate)) goto pop_3_error;
                 attrs = Py_None;  // Failure!
             }
-            #line 1646 "Python/executor_cases.c.h"
+            #line 1667 "Python/executor_cases.c.h"
             STACK_SHRINK(2);
             stack_pointer[-1] = attrs;
             break;
@@ -1654,7 +1675,7 @@
             #line 2327 "Python/bytecodes.c"
             int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
             res = match ? Py_True : Py_False;
-            #line 1658 "Python/executor_cases.c.h"
+            #line 1679 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = res;
             break;
@@ -1666,7 +1687,7 @@
             #line 2332 "Python/bytecodes.c"
             int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
             res = match ? Py_True : Py_False;
-            #line 1670 "Python/executor_cases.c.h"
+            #line 1691 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = res;
             break;
@@ -1680,7 +1701,7 @@
             // On successful match, PUSH(values). Otherwise, PUSH(None).
             values_or_none = match_keys(tstate, subject, keys);
             if (values_or_none == NULL) goto error;
-            #line 1684 "Python/executor_cases.c.h"
+            #line 1705 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = values_or_none;
             break;
@@ -1692,11 +1713,11 @@
             #line 2343 "Python/bytecodes.c"
             /* before: [obj]; after [getiter(obj)] */
             iter = PyObject_GetIter(iterable);
-            #line 1696 "Python/executor_cases.c.h"
+            #line 1717 "Python/executor_cases.c.h"
             Py_DECREF(iterable);
             #line 2346 "Python/bytecodes.c"
             if (iter == NULL) goto pop_1_error;
-            #line 1700 "Python/executor_cases.c.h"
+            #line 1721 "Python/executor_cases.c.h"
             stack_pointer[-1] = iter;
             break;
         }
@@ -1727,11 +1748,11 @@
                 if (iter == NULL) {
                     goto error;
                 }
-            #line 1731 "Python/executor_cases.c.h"
+            #line 1752 "Python/executor_cases.c.h"
                 Py_DECREF(iterable);
             #line 2373 "Python/bytecodes.c"
             }
-            #line 1735 "Python/executor_cases.c.h"
+            #line 1756 "Python/executor_cases.c.h"
             stack_pointer[-1] = iter;
             break;
         }
@@ -1762,7 +1783,7 @@
             res = PyObject_Vectorcall(exit_func, stack + 1,
                     3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
             if (res == NULL) goto error;
-            #line 1766 "Python/executor_cases.c.h"
+            #line 1787 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = res;
             break;
@@ -1781,7 +1802,7 @@
             }
             assert(PyExceptionInstance_Check(new_exc));
             exc_info->exc_value = Py_NewRef(new_exc);
-            #line 1785 "Python/executor_cases.c.h"
+            #line 1806 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = new_exc;
             stack_pointer[-2] = prev_exc;
@@ -1798,7 +1819,7 @@
                     Py_TYPE(should_be_none)->tp_name);
                 goto error;
             }
-            #line 1802 "Python/executor_cases.c.h"
+            #line 1823 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             break;
         }
@@ -1818,7 +1839,7 @@
 
             func_obj->func_version = ((PyCodeObject *)codeobj)->co_version;
             func = (PyObject *)func_obj;
-            #line 1822 "Python/executor_cases.c.h"
+            #line 1843 "Python/executor_cases.c.h"
             stack_pointer[-1] = func;
             break;
         }
@@ -1851,7 +1872,7 @@
                 default:
                     Py_UNREACHABLE();
             }
-            #line 1855 "Python/executor_cases.c.h"
+            #line 1876 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = func;
             break;
@@ -1864,13 +1885,13 @@
             PyObject *slice;
             #line 3491 "Python/bytecodes.c"
             slice = PySlice_New(start, stop, step);
-            #line 1868 "Python/executor_cases.c.h"
+            #line 1889 "Python/executor_cases.c.h"
             Py_DECREF(start);
             Py_DECREF(stop);
             Py_XDECREF(step);
             #line 3493 "Python/bytecodes.c"
             if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; }
-            #line 1874 "Python/executor_cases.c.h"
+            #line 1895 "Python/executor_cases.c.h"
             STACK_SHRINK(((oparg == 3) ? 1 : 0));
             STACK_SHRINK(1);
             stack_pointer[-1] = slice;
@@ -1887,7 +1908,7 @@
             result = conv_fn(value);
             Py_DECREF(value);
             if (result == NULL) goto pop_1_error;
-            #line 1891 "Python/executor_cases.c.h"
+            #line 1912 "Python/executor_cases.c.h"
             stack_pointer[-1] = result;
             break;
         }
@@ -1906,7 +1927,7 @@
             else {
                 res = value;
             }
-            #line 1910 "Python/executor_cases.c.h"
+            #line 1931 "Python/executor_cases.c.h"
             stack_pointer[-1] = res;
             break;
         }
@@ -1920,7 +1941,7 @@
             Py_DECREF(value);
             Py_DECREF(fmt_spec);
             if (res == NULL) goto pop_2_error;
-            #line 1924 "Python/executor_cases.c.h"
+            #line 1945 "Python/executor_cases.c.h"
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             break;
@@ -1932,7 +1953,7 @@
             #line 3526 "Python/bytecodes.c"
             assert(oparg > 0);
             top = Py_NewRef(bottom);
-            #line 1936 "Python/executor_cases.c.h"
+            #line 1957 "Python/executor_cases.c.h"
             STACK_GROW(1);
             stack_pointer[-1] = top;
             break;
@@ -1943,7 +1964,7 @@
             PyObject *bottom = stack_pointer[-(2 + (oparg-2))];
             #line 3551 "Python/bytecodes.c"
             assert(oparg >= 2);
-            #line 1947 "Python/executor_cases.c.h"
+            #line 1968 "Python/executor_cases.c.h"
             stack_pointer[-1] = bottom;
             stack_pointer[-(2 + (oparg-2))] = top;
             break;

diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h
index 6a42775e091208056570ae3717a4d83bdcc836e1..ac3d800d8fe0fe6e4870938cb6111a3dd37f2b0f 100644 (file)
--- a/Python/opcode_metadata.h
+++ b/Python/opcode_metadata.h
@@ -20,7 +20,7 @@
     0)
 
 #define EXIT_TRACE 300
-#define SET_IP 301
+#define SAVE_IP 301
 #define _GUARD_BOTH_INT 302
 #define _BINARY_OP_MULTIPLY_INT 303
 #define _BINARY_OP_ADD_INT 304
@@ -1164,6 +1164,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = {
 };
 const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
     [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } },
+    [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } },
     [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } },
     [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } },
     [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } },
@@ -1218,6 +1219,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
     [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } },
     [LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } },
     [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } },
+    [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } },
     [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } },
     [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } },
     [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } },
@@ -1266,7 +1268,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
 #ifdef Py_DEBUG
 const char * const _PyOpcode_uop_name[512] = {
     [300] = "EXIT_TRACE",
-    [301] = "SET_IP",
+    [301] = "SAVE_IP",
     [302] = "_GUARD_BOTH_INT",
     [303] = "_BINARY_OP_MULTIPLY_INT",
     [304] = "_BINARY_OP_ADD_INT",

diff --git a/Python/optimizer.c b/Python/optimizer.c
index b00825ade16e0710ac2fd48c2b1b264d17c10ce1..32f0b1477d203cb1835d6e0bde7ea5e103e1d72e 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -282,11 +282,6 @@ PyUnstable_Optimizer_NewCounter(void)
 
 ///////////////////// Experimental UOp Optimizer /////////////////////
 
-#ifdef Py_DEBUG
-   /* For debugging the interpreter: */
-#  define LLTRACE  1      /* Low-level trace feature */
-#endif
-
 static void
 uop_dealloc(_PyUOpExecutorObject *self) {
     PyObject_Free(self);
@@ -308,60 +303,81 @@ translate_bytecode_to_trace(
     _PyUOpInstruction *trace,
     int max_length)
 {
-#ifdef LLTRACE
+    int trace_length = 0;
+
+#ifdef Py_DEBUG
     char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
     int lltrace = 0;
     if (uop_debug != NULL && *uop_debug >= '0') {
         lltrace = *uop_debug - '0';  // TODO: Parse an int and all that
     }
-    if (lltrace >= 4) {
-        fprintf(stderr,
-                "Optimizing %s (%s:%d) at offset %ld\n",
-                PyUnicode_AsUTF8(code->co_qualname),
-                PyUnicode_AsUTF8(code->co_filename),
-                code->co_firstlineno,
-                (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
-    }
-#define ADD_TO_TRACE(OPCODE, OPERAND) \
-        if (lltrace >= 2) { \
-            const char *opname = (OPCODE) < 256 ? _PyOpcode_OpName[(OPCODE)] : _PyOpcode_uop_name[(OPCODE)]; \
-            fprintf(stderr, "  ADD_TO_TRACE(%s, %" PRIu64 ")\n", opname, (uint64_t)(OPERAND)); \
-        } \
-        trace[trace_length].opcode = (OPCODE); \
-        trace[trace_length].operand = (OPERAND); \
-        trace_length++;
+#define DPRINTF(level, ...) \
+    if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); }
 #else
-#define ADD_TO_TRACE(OPCODE, OPERAND) \
-        trace[trace_length].opcode = (OPCODE); \
-        trace[trace_length].operand = (OPERAND); \
-        trace_length++;
+#define DPRINTF(level, ...)
 #endif
 
-    int trace_length = 0;
-    // Always reserve space for one uop, plus SET_UP, plus EXIT_TRACE
-    while (trace_length + 3 <= max_length) {
+#define ADD_TO_TRACE(OPCODE, OPERAND) \
+    DPRINTF(2, \
+            "  ADD_TO_TRACE(%s, %" PRIu64 ")\n", \
+            (OPCODE) < 256 ? _PyOpcode_OpName[(OPCODE)] : _PyOpcode_uop_name[(OPCODE)], \
+            (uint64_t)(OPERAND)); \
+    assert(trace_length < max_length); \
+    trace[trace_length].opcode = (OPCODE); \
+    trace[trace_length].operand = (OPERAND); \
+    trace_length++;
+
+    DPRINTF(4,
+            "Optimizing %s (%s:%d) at offset %ld\n",
+            PyUnicode_AsUTF8(code->co_qualname),
+            PyUnicode_AsUTF8(code->co_filename),
+            code->co_firstlineno,
+            (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
+
+    for (;;) {
+        ADD_TO_TRACE(SAVE_IP, (int)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
         int opcode = instr->op.code;
         uint64_t operand = instr->op.arg;
         switch (opcode) {
             case LOAD_FAST_LOAD_FAST:
+            case STORE_FAST_LOAD_FAST:
+            case STORE_FAST_STORE_FAST:
             {
-                // Reserve space for two uops (+ SETUP + EXIT_TRACE)
+                // Reserve space for two uops (+ SAVE_IP + EXIT_TRACE)
                 if (trace_length + 4 > max_length) {
+                    DPRINTF(1, "Ran out of space for LOAD_FAST_LOAD_FAST\n");
                     goto done;
                 }
                 uint64_t oparg1 = operand >> 4;
                 uint64_t oparg2 = operand & 15;
-                ADD_TO_TRACE(LOAD_FAST, oparg1);
-                ADD_TO_TRACE(LOAD_FAST, oparg2);
+                switch (opcode) {
+                    case LOAD_FAST_LOAD_FAST:
+                        ADD_TO_TRACE(LOAD_FAST, oparg1);
+                        ADD_TO_TRACE(LOAD_FAST, oparg2);
+                        break;
+                    case STORE_FAST_LOAD_FAST:
+                        ADD_TO_TRACE(STORE_FAST, oparg1);
+                        ADD_TO_TRACE(LOAD_FAST, oparg2);
+                        break;
+                    case STORE_FAST_STORE_FAST:
+                        ADD_TO_TRACE(STORE_FAST, oparg1);
+                        ADD_TO_TRACE(STORE_FAST, oparg2);
+                        break;
+                    default:
+                        Py_FatalError("Missing case");
+                }
                 break;
             }
             default:
             {
                 const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];
                 if (expansion->nuops > 0) {
-                    // Reserve space for nuops (+ SETUP + EXIT_TRACE)
+                    // Reserve space for nuops (+ SAVE_IP + EXIT_TRACE)
                     int nuops = expansion->nuops;
                     if (trace_length + nuops + 2 > max_length) {
+                        DPRINTF(1,
+                                "Ran out of space for %s\n",
+                                opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]);
                         goto done;
                     }
                     for (int i = 0; i < nuops; i++) {
@@ -387,49 +403,45 @@ translate_bytecode_to_trace(
                                 Py_FatalError("garbled expansion");
                         }
                         ADD_TO_TRACE(expansion->uops[i].uop, operand);
-                        assert(expansion->uops[0].size == 0);  // TODO
                     }
                     break;
                 }
-                // fprintf(stderr, "Unsupported opcode %d\n", opcode);
-                goto done;  // Break out of while loop
+                DPRINTF(2,
+                        "Unsupported opcode %s\n",
+                        opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]);
+                goto done;  // Break out of loop
             }
         }
         instr++;
         // Add cache size for opcode
         instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
-        ADD_TO_TRACE(SET_IP, (int)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
     }
+
 done:
-    if (trace_length > 0) {
+    // Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE
+    if (trace_length > 3) {
         ADD_TO_TRACE(EXIT_TRACE, 0);
-#ifdef LLTRACE
-        if (lltrace >= 1) {
-            fprintf(stderr,
-                    "Created a trace for %s (%s:%d) at offset %ld -- length %d\n",
-                    PyUnicode_AsUTF8(code->co_qualname),
-                    PyUnicode_AsUTF8(code->co_filename),
-                    code->co_firstlineno,
-                    (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive),
-                    trace_length);
-        }
-#endif
+        DPRINTF(1,
+                "Created a trace for %s (%s:%d) at offset %ld -- length %d\n",
+                PyUnicode_AsUTF8(code->co_qualname),
+                PyUnicode_AsUTF8(code->co_filename),
+                code->co_firstlineno,
+                (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive),
+                trace_length);
+        return trace_length;
     }
     else {
-#ifdef LLTRACE
-        if (lltrace >= 4) {
-            fprintf(stderr,
-                    "No trace for %s (%s:%d) at offset %ld\n",
-                    PyUnicode_AsUTF8(code->co_qualname),
-                    PyUnicode_AsUTF8(code->co_filename),
-                    code->co_firstlineno,
-                    (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
-        }
-#endif
+        DPRINTF(4,
+                "No trace for %s (%s:%d) at offset %ld\n",
+                PyUnicode_AsUTF8(code->co_qualname),
+                PyUnicode_AsUTF8(code->co_filename),
+                code->co_firstlineno,
+                (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
     }
-    return trace_length;
+    return 0;
 
 #undef ADD_TO_TRACE
+#undef DPRINTF
 }
 
 static int

diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py
index 38be98034e0d83a88a5f9a02cf552ec82ee024c2..657dfa93fd537d34e9bfbd757b7f266aba7d53ee 100644 (file)
--- a/Tools/cases_generator/generate_cases.py
+++ b/Tools/cases_generator/generate_cases.py
@@ -308,8 +308,7 @@ class ActiveCacheEffect:
 
 
 FORBIDDEN_NAMES_IN_UOPS = (
-    "resume_with_error",  # Proxy for "goto", which isn't an IDENTIFIER
-    "unbound_local_error",
+    "resume_with_error",
     "kwnames",
     "next_instr",
     "oparg1",  # Proxy for super-instructions like LOAD_FAST_LOAD_FAST
@@ -401,20 +400,25 @@ class Instruction:
     def is_viable_uop(self) -> bool:
         """Whether this instruction is viable as a uop."""
         if self.always_exits:
+            # print(f"Skipping {self.name} because it always exits")
             return False
         if self.instr_flags.HAS_ARG_FLAG:
             # If the instruction uses oparg, it cannot use any caches
             if self.active_caches:
+                # print(f"Skipping {self.name} because it uses oparg and caches")
                 return False
         else:
             # If it doesn't use oparg, it can have one cache entry
             if len(self.active_caches) > 1:
+                # print(f"Skipping {self.name} because it has >1 cache entries")
                 return False
+        res = True
         for forbidden in FORBIDDEN_NAMES_IN_UOPS:
             # TODO: Don't check in '#ifdef ENABLE_SPECIALIZATION' regions
             if variable_used(self.inst, forbidden):
-                return False
-        return True
+                # print(f"Skipping {self.name} because it uses {forbidden}")
+                res = False
+        return res
 
     def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None:
         """Write one instruction, sans prologue and epilogue."""
@@ -1323,7 +1327,7 @@ class Analyzer:
             self.out.emit(make_text(name, counter))
             counter += 1
         add("EXIT_TRACE")
-        add("SET_IP")
+        add("SAVE_IP")
         for instr in self.instrs.values():
             if instr.kind == "op" and instr.is_viable_uop():
                 add(instr.name)