GH-100762: Don't call `gen.throw()` in `gen.close()`, unless necessary. (GH-101013)

* Store exception stack depth in YIELD_VALUE's oparg and use it avoid expensive gen.throw() in gen.close() where possible.

diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 7f309ce56ddec0ffef607598b9bb4e750f1b85c8..6a68ec4b14be31943b7790cbf12151dab963d78d 100644 (file)
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -700,7 +700,10 @@ iterations of the loop.
    Yields ``STACK.pop()`` from a :term:`generator`.
 
     .. versionchanged:: 3.11
-       oparg set to be the stack depth, for efficient handling on frames.
+       oparg set to be the stack depth.
+
+    .. versionchanged:: 3.12
+       oparg set to be the exception block depth, for efficient closing of generators.
 
 
 .. opcode:: SETUP_ANNOTATIONS

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 5a8cbb98ed3f5668e33b314ca1231f4c0971da4c..cd44d0050ede82ee5139047d04b4e4f68fc7a833 100644 (file)
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -429,7 +429,8 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.12a1 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR)
 #     Python 3.12a1 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE)
 #     Python 3.12a1 3515 (Embed jump mask in COMPARE_OP oparg)
-#     Python 3.12a1 3516 (Add COMAPRE_AND_BRANCH instruction)
+#     Python 3.12a1 3516 (Add COMPARE_AND_BRANCH instruction)
+#     Python 3.12a1 3517 (Change YIELD_VALUE oparg to exception block depth)
 
 #     Python 3.13 will start with 3550
 
@@ -442,7 +443,7 @@ _code_type = type(_write_atomic.__code__)
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3517).to_bytes(2, 'little') + b'\r\n'
 
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 

diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index 27487de8fb3d81c13d64513ab81ae66147adf71c..bdf48c15309296caf7e317656346fa9d1aa692b3 100644 (file)
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -492,7 +492,7 @@ dis_asyncwith = """\
            GET_AWAITABLE            1
            LOAD_CONST               0 (None)
         >> SEND                     3 (to 22)
-           YIELD_VALUE              3
+           YIELD_VALUE              2
            RESUME                   3
            JUMP_BACKWARD_NO_INTERRUPT     4 (to 14)
         >> POP_TOP
@@ -526,7 +526,7 @@ dis_asyncwith = """\
            GET_AWAITABLE            2
            LOAD_CONST               0 (None)
         >> SEND                     4 (to 92)
-           YIELD_VALUE              6
+           YIELD_VALUE              3
            RESUME                   3
            JUMP_BACKWARD_NO_INTERRUPT     4 (to 82)
         >> CLEANUP_THROW

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst
new file mode 100644 (file)
index 0000000..2f6b121
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst
@@ -0,0 +1,3 @@
+Record the (virtual) exception block depth in the oparg of
+:opcode:`YIELD_VALUE`. Use this to avoid the expensive ``throw()`` when
+closing generators (and coroutines) that can be closed trivially.

diff --git a/Objects/genobject.c b/Objects/genobject.c
index 2adb1e4bf308e46fa0e8dfdc7dd6fafc7a7dee24..35246653c453488c454fd99fd5b2412bfb5933fb 100644 (file)
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -354,6 +354,13 @@ gen_close(PyGenObject *gen, PyObject *args)
     PyObject *yf = _PyGen_yf(gen);
     int err = 0;
 
+    if (gen->gi_frame_state == FRAME_CREATED) {
+        gen->gi_frame_state = FRAME_COMPLETED;
+        Py_RETURN_NONE;
+    }
+    if (gen->gi_frame_state >= FRAME_COMPLETED) {
+        Py_RETURN_NONE;
+    }
     if (yf) {
         PyFrameState state = gen->gi_frame_state;
         gen->gi_frame_state = FRAME_EXECUTING;
@@ -361,8 +368,23 @@ gen_close(PyGenObject *gen, PyObject *args)
         gen->gi_frame_state = state;
         Py_DECREF(yf);
     }
-    if (err == 0)
+    _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe;
+    /* It is possible for the previous instruction to not be a
+     * YIELD_VALUE if the debugger has changed the lineno. */
+    if (err == 0 && frame->prev_instr->opcode == YIELD_VALUE) {
+        assert(frame->prev_instr[1].opcode == RESUME);
+        int exception_handler_depth = frame->prev_instr->oparg;
+        assert(exception_handler_depth > 0);
+        /* We can safely ignore the outermost try block
+         * as it automatically generated to handle
+         * StopIteration. */
+        if (exception_handler_depth == 1) {
+            Py_RETURN_NONE;
+        }
+    }
+    if (err == 0) {
         PyErr_SetNone(PyExc_GeneratorExit);
+    }
     retval = gen_send_ex(gen, Py_None, 1, 1);
     if (retval) {
         const char *msg = "generator ignored GeneratorExit";

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 47bbe1a99e3bf0d7d9418a8d3ef4fbf4194452f2..ac54791d67e4398e2526e55beaaa857d45b3e1a2 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -720,7 +720,6 @@ dummy_func(
             // NOTE: It's important that YIELD_VALUE never raises an exception!
             // The compiler treats any exception raised here as a failed close()
             // or throw() call.
-            assert(oparg == STACK_LEVEL());
             assert(frame != &entry_frame);
             PyGenObject *gen = _PyFrame_GetGenerator(frame);
             gen->gi_frame_state = FRAME_SUSPENDED;

diff --git a/Python/compile.c b/Python/compile.c
index ce714dce6edfe656f1ddf8ac7132b82b10ad9f21..9fc997cdf525e9f204c38fe11cb230bc34e880f9 100644 (file)
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -7162,9 +7162,6 @@ stackdepth(basicblock *entryblock, int code_flags)
                 next = NULL;
                 break;
             }
-            if (instr->i_opcode == YIELD_VALUE) {
-                instr->i_oparg = depth;
-            }
         }
         if (next != NULL) {
             assert(BB_HAS_FALLTHROUGH(b));
@@ -7332,6 +7329,9 @@ label_exception_targets(basicblock *entryblock) {
                 }
             }
             else {
+                if (instr->i_opcode == YIELD_VALUE) {
+                    instr->i_oparg = except_stack->depth;
+                }
                 instr->i_except = handler;
             }
         }

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index c1eb4000883dc794e750e433041aadeab874c5ad..5dcc8eeec19f896e98841063d826646073c584da 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -919,7 +919,6 @@
             // NOTE: It's important that YIELD_VALUE never raises an exception!
             // The compiler treats any exception raised here as a failed close()
             // or throw() call.
-            assert(oparg == STACK_LEVEL());
             assert(frame != &entry_frame);
             PyGenObject *gen = _PyFrame_GetGenerator(frame);
             gen->gi_frame_state = FRAME_SUSPENDED;

diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h
index 3ceaca8c397f6b01a74be6a021940f41e06c6008..1fb0acceb511c747d7de60305d4dee198bec8380 100644 (file)
--- a/Python/opcode_metadata.h
+++ b/Python/opcode_metadata.h
@@ -59,7 +59,7 @@ static const struct {
     [GET_ANEXT] = { 1, 2, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },
     [GET_AWAITABLE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
     [SEND] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
-    [YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
+    [YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },
     [POP_EXCEPT] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },
     [RERAISE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
     [PREP_RERAISE_STAR] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },