gh-151907: Avoid creating temporary objects in some list comprehensions (GH-151908)

Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index fdef9803144b44406bd2cbf68e80d93acb5f571f..ecc69b5383e3f21166d587aa02cce3d647299e0e 100644 (file)
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -2063,7 +2063,7 @@ class TestSourcePositions(unittest.TestCase):
 
     def test_multiline_list_comprehension(self):
         snippet = textwrap.dedent("""\
-            [(x,
+            _ = [(x,
                 2*x)
                 for x
                 in [1,2,3] if (x > 0
@@ -2073,14 +2073,14 @@ class TestSourcePositions(unittest.TestCase):
         compiled_code, _ = self.check_positions_against_ast(snippet)
         self.assertIsInstance(compiled_code, types.CodeType)
         self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
-            line=1, end_line=2, column=1, end_column=8, occurrence=1)
+            line=1, end_line=2, column=5, end_column=8, occurrence=1)
         self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
-            line=1, end_line=2, column=1, end_column=8, occurrence=1)
+            line=1, end_line=2, column=5, end_column=8, occurrence=1)
 
     def test_multiline_async_list_comprehension(self):
         snippet = textwrap.dedent("""\
             async def f():
-                [(x,
+                _ = [(x,
                     2*x)
                     async for x
                     in [1,2,3] if (x > 0
@@ -2093,11 +2093,11 @@ class TestSourcePositions(unittest.TestCase):
         compiled_code = g['f'].__code__
         self.assertIsInstance(compiled_code, types.CodeType)
         self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
-            line=2, end_line=3, column=5, end_column=12, occurrence=1)
+            line=2, end_line=3, column=9, end_column=12, occurrence=1)
         self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
-            line=2, end_line=3, column=5, end_column=12, occurrence=1)
+            line=2, end_line=3, column=9, end_column=12, occurrence=1)
         self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
-            line=2, end_line=7, column=4, end_column=36, occurrence=1)
+            line=2, end_line=2, column=4, end_column=5, occurrence=1)
 
     def test_multiline_set_comprehension(self):
         snippet = textwrap.dedent("""\

diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py
index 8831af473c9d3fb97cf27364a46540b77f7ec362..87509245f6a866c187e3c2c0bca96f2b9095a6ce 100644 (file)
--- a/Lib/test/test_compiler_codegen.py
+++ b/Lib/test/test_compiler_codegen.py
@@ -216,3 +216,39 @@ class IsolatedCodeGenTests(CodegenTestCase):
             ('RETURN_VALUE', None)
         ]
         self.codegen_test(snippet, expected)
+
+    def test_comp_without_target_optimization(self):
+        snippet = "[i for i in range(10)]"
+        expected = [
+            ('RESUME', 0),
+            ('ANNOTATIONS_PLACEHOLDER', None),
+            ('LOAD_NAME', 0),
+            ('PUSH_NULL', None),
+            ('LOAD_CONST', 0),
+            ('CALL', 1),
+            ('LOAD_FAST_AND_CLEAR', 0),
+            ('SWAP', 2),
+            ('SETUP_FINALLY', 20),
+            ('COPY', 1),
+            ('GET_ITER', 0),
+            ('FOR_ITER', 16),
+            ('STORE_FAST', 0),
+            ('LOAD_FAST', 0),
+            ('POP_TOP', None),
+            ('JUMP', 11),
+            ('END_FOR', None),
+            ('POP_ITER', None),
+            ('POP_BLOCK', None),
+            ('JUMP_NO_INTERRUPT', 25),
+            ('SWAP', 2),
+            ('POP_TOP', None),
+            ('SWAP', 2),
+            ('STORE_FAST_MAYBE_NULL', 0),
+            ('RERAISE', 0),
+            ('SWAP', 2),
+            ('STORE_FAST_MAYBE_NULL', 0),
+            ('POP_TOP', None),
+            ('LOAD_CONST', 1),
+            ('RETURN_VALUE', None),
+        ]
+        self.codegen_test(snippet, expected)

diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py
index a7a46216787437ea0e27d252b378353a76217651..9873e1f80aa631d8844806f43563beccb36324ca 100644 (file)
--- a/Lib/test/test_dictcomps.py
+++ b/Lib/test/test_dictcomps.py
@@ -165,6 +165,14 @@ class DictComprehensionTest(unittest.TestCase):
                 self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
                                  expected)
 
+    def test_hash_error(self):
+        class Unhashable:
+            def __hash__(self):
+                0/0
+
+        with self.assertRaises(ZeroDivisionError):
+            {unhashable: 1 for unhashable in [Unhashable()]}
+
 
 if __name__ == "__main__":
     unittest.main()

diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py
index cf3796d9480801598103051b27db63c726454de0..fca9acbc6b1ef6c4b0d5642db638dae697f96082 100644 (file)
--- a/Lib/test/test_listcomps.py
+++ b/Lib/test/test_listcomps.py
@@ -793,6 +793,75 @@ class ListComprehensionTest(unittest.TestCase):
                 self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
                                  expected)
 
+    def test_optimization_with_side_effects(self):
+        # List comprehensions that aren't used as a value are optimized
+        # to avoid creating a list. Ensure that side effects are still
+        # retained when this happens.
+        with self.assertRaises(ZeroDivisionError):
+            [0/0 for _ in [1]]
+
+        count = 0
+        def increment():
+            nonlocal count
+            count += 1
+
+        [increment() for _ in range(5)]
+        self.assertEqual(count, 5)
+
+    def test_async_optimization_with_side_effects(self):
+        async def gen1(aiterator):
+            with self.assertRaises(ZeroDivisionError):
+                [0/0 async for _ in aiterator]
+
+        async def gen2(aiterator):
+            [increment() async for _ in aiterator]
+
+        async def numbers():
+            for i in range(5):
+                yield i
+
+        count = 0
+        def increment():
+            nonlocal count
+            count += 1
+
+        def exhaust(coro):
+            try:
+                coro.send(None)
+            except StopIteration:
+                pass
+
+        exhaust(gen1(numbers()))
+        exhaust(gen2(numbers()))
+        self.assertEqual(count, 5)
+
+    def test_optimization_with_starred_unpack(self):
+        with self.assertRaises(TypeError):
+            [*i for i in [1, 2, 3]]
+
+        async def coro():
+            async def gen():
+                yield 1
+
+            with self.assertRaises(TypeError):
+                [*i async for i in gen()]
+
+        c = coro()
+        while True:
+            try:
+                c.send(None)
+            except StopIteration:
+                break
+
+        count = 0
+        def weird():
+            nonlocal count
+            count += 1
+            yield 0
+
+        [*weird() for _ in range(5)]
+        self.assertEqual(count, 5)
+
 __test__ = {'doctests' : doctests}
 
 def load_tests(loader, tests, pattern):

diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py
index 6fc5bb74036c5fa1d645dbf8f0b63f0071b44b87..27748ebd997a620a1248a237c63f7b2c0896cd87 100644 (file)
--- a/Lib/test/test_setcomps.py
+++ b/Lib/test/test_setcomps.py
@@ -188,6 +188,18 @@ class SetComprehensionTest(unittest.TestCase):
                 self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
                                  expected)
 
+    def test_hash_error(self):
+        class Unhashable:
+            def __hash__(self):
+                0/0
+
+        with self.assertRaises(ZeroDivisionError):
+            {unhashable for unhashable in [Unhashable()]}
+
+
+if __name__ == "__main__":
+    unittest.main()
+
 __test__ = {'doctests' : doctests}
 
 def load_tests(loader, tests, pattern):

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst
new file mode 100644 (file)
index 0000000..cca300f
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst
@@ -0,0 +1,2 @@
+Avoid creating :class:`list` objects in comprehensions when the comprehension
+is not used as a value.

diff --git a/Python/codegen.c b/Python/codegen.c
index 81fecd09a2060d18a4d70f23192537fe53b4bf06..f2c2b21d106fbdd17fd73b93875d526acb7d922f 100644 (file)
--- a/Python/codegen.c
+++ b/Python/codegen.c
@@ -200,6 +200,7 @@ static int codegen_nameop(compiler *, location, identifier, expr_context_ty);
 static int codegen_visit_stmt(compiler *, stmt_ty);
 static int codegen_visit_keyword(compiler *, keyword_ty);
 static int codegen_visit_expr(compiler *, expr_ty);
+static int codegen_visit_unused_expr(compiler *, expr_ty);
 static int codegen_augassign(compiler *, stmt_ty);
 static int codegen_annassign(compiler *, stmt_ty);
 static int codegen_subscript(compiler *, expr_ty);
@@ -238,14 +239,14 @@ static int codegen_sync_comprehension_generator(
                                       asdl_comprehension_seq *generators, int gen_index,
                                       int depth,
                                       expr_ty elt, expr_ty val, int type,
-                                      IterStackPosition iter_pos);
+                                      IterStackPosition iter_pos, bool avoid_creation);
 
 static int codegen_async_comprehension_generator(
                                       compiler *c, location loc,
                                       asdl_comprehension_seq *generators, int gen_index,
                                       int depth,
                                       expr_ty elt, expr_ty val, int type,
-                                      IterStackPosition iter_pos);
+                                      IterStackPosition iter_pos, bool avoid_creation);
 
 static int codegen_pattern(compiler *, pattern_ty, pattern_context *);
 static int codegen_match(compiler *, stmt_ty);
@@ -475,6 +476,9 @@ codegen_addop_j(instr_sequence *seq, location loc,
         }                                                                   \
     } while (0)
 
+#define VISIT_UNUSED(C, TYPE, V) \
+    RETURN_IF_ERROR(codegen_visit_unused_ ## TYPE((C), (V)))
+
 static int
 codegen_call_exit_with_nones(compiler *c, location loc)
 {
@@ -3084,7 +3088,7 @@ codegen_stmt_expr(compiler *c, location loc, expr_ty value)
         return SUCCESS;
     }
 
-    VISIT(c, expr, value);
+    VISIT_UNUSED(c, expr, value);
     ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */
     return SUCCESS;
 }
@@ -4547,27 +4551,47 @@ codegen_comprehension_generator(compiler *c, location loc,
                                 asdl_comprehension_seq *generators, int gen_index,
                                 int depth,
                                 expr_ty elt, expr_ty val, int type,
-                                IterStackPosition iter_pos)
+                                IterStackPosition iter_pos, bool avoid_creation)
 {
     comprehension_ty gen;
     gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
     if (gen->is_async) {
         return codegen_async_comprehension_generator(
             c, loc, generators, gen_index, depth, elt, val, type,
-            iter_pos);
+            iter_pos, avoid_creation);
     } else {
         return codegen_sync_comprehension_generator(
             c, loc, generators, gen_index, depth, elt, val, type,
-            iter_pos);
+            iter_pos, avoid_creation);
     }
 }
 
+static int
+codegen_unpack_starred(compiler *c, location loc, expr_ty value, bool yield)
+{
+    NEW_JUMP_TARGET_LABEL(c, unpack_start);
+    NEW_JUMP_TARGET_LABEL(c, unpack_end);
+    VISIT(c, expr, value);
+    ADDOP_I(c, loc, GET_ITER, 0);
+    USE_LABEL(c, unpack_start);
+    ADDOP_JUMP(c, loc, FOR_ITER, unpack_end);
+    if (yield) {
+        ADDOP_YIELD(c, loc);
+    }
+    ADDOP(c, loc, POP_TOP);
+    ADDOP_JUMP(c, NO_LOCATION, JUMP, unpack_start);
+    USE_LABEL(c, unpack_end);
+    ADDOP(c, NO_LOCATION, END_FOR);
+    ADDOP(c, NO_LOCATION, POP_ITER);
+    return SUCCESS;
+}
+
 static int
 codegen_sync_comprehension_generator(compiler *c, location loc,
                                      asdl_comprehension_seq *generators,
                                      int gen_index, int depth,
                                      expr_ty elt, expr_ty val, int type,
-                                     IterStackPosition iter_pos)
+                                     IterStackPosition iter_pos, bool avoid_creation)
 {
     /* generate code for the iterator, then each of the ifs,
        and then write to the element */
@@ -4635,7 +4659,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
         RETURN_IF_ERROR(
             codegen_comprehension_generator(c, loc,
                                             generators, gen_index, depth,
-                                            elt, val, type, ITERABLE_IN_LOCAL));
+                                            elt, val, type, ITERABLE_IN_LOCAL, avoid_creation));
     }
 
     location elt_loc = LOC(elt);
@@ -4645,19 +4669,9 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
         /* comprehension specific code */
         switch (type) {
         case COMP_GENEXP:
+            assert(!avoid_creation);
             if (elt->kind == Starred_kind) {
-                NEW_JUMP_TARGET_LABEL(c, unpack_start);
-                NEW_JUMP_TARGET_LABEL(c, unpack_end);
-                VISIT(c, expr, elt->v.Starred.value);
-                ADDOP_I(c, elt_loc, GET_ITER, 0);
-                USE_LABEL(c, unpack_start);
-                ADDOP_JUMP(c, elt_loc, FOR_ITER, unpack_end);
-                ADDOP_YIELD(c, elt_loc);
-                ADDOP(c, elt_loc, POP_TOP);
-                ADDOP_JUMP(c, NO_LOCATION, JUMP, unpack_start);
-                USE_LABEL(c, unpack_end);
-                ADDOP(c, NO_LOCATION, END_FOR);
-                ADDOP(c, NO_LOCATION, POP_ITER);
+                RETURN_IF_ERROR(codegen_unpack_starred(c, elt_loc, elt->v.Starred.value, /*yield=*/true));
             }
             else {
                 VISIT(c, expr, elt);
@@ -4666,6 +4680,15 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
             }
             break;
         case COMP_LISTCOMP:
+            if (avoid_creation) {
+                if (elt->kind == Starred_kind) {
+                    RETURN_IF_ERROR(codegen_unpack_starred(c, elt_loc, elt->v.Starred.value, /*yield=*/false));
+                } else {
+                    VISIT(c, expr, elt);
+                    ADDOP(c, elt_loc, POP_TOP);
+                }
+                break;
+            }
             if (elt->kind == Starred_kind) {
                 VISIT(c, expr, elt->v.Starred.value);
                 ADDOP_I(c, elt_loc, LIST_EXTEND, depth + 1);
@@ -4729,7 +4752,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
                                       asdl_comprehension_seq *generators,
                                       int gen_index, int depth,
                                       expr_ty elt, expr_ty val, int type,
-                                      IterStackPosition iter_pos)
+                                      IterStackPosition iter_pos, bool avoid_creation)
 {
     NEW_JUMP_TARGET_LABEL(c, start);
     NEW_JUMP_TARGET_LABEL(c, send);
@@ -4779,7 +4802,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
         RETURN_IF_ERROR(
             codegen_comprehension_generator(c, loc,
                                             generators, gen_index, depth,
-                                            elt, val, type, 0));
+                                            elt, val, type, 0, avoid_creation));
     }
 
     location elt_loc = LOC(elt);
@@ -4788,6 +4811,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
         /* comprehension specific code */
         switch (type) {
         case COMP_GENEXP:
+            assert(!avoid_creation);
             if (elt->kind == Starred_kind) {
                 NEW_JUMP_TARGET_LABEL(c, unpack_start);
                 NEW_JUMP_TARGET_LABEL(c, unpack_end);
@@ -4809,6 +4833,16 @@ codegen_async_comprehension_generator(compiler *c, location loc,
             }
             break;
         case COMP_LISTCOMP:
+            if (avoid_creation) {
+                if (elt->kind == Starred_kind) {
+                    RETURN_IF_ERROR(codegen_unpack_starred(c, elt_loc, elt->v.Starred.value, /*yield=*/false));
+                } else {
+                    VISIT(c, expr, elt);
+                    ADDOP(c, elt_loc, POP_TOP);
+                }
+                break;
+            }
+
             if (elt->kind == Starred_kind) {
                 VISIT(c, expr, elt->v.Starred.value);
                 ADDOP_I(c, elt_loc, LIST_EXTEND, depth + 1);
@@ -4819,6 +4853,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
             }
             break;
         case COMP_SETCOMP:
+            assert(!avoid_creation);
             if (elt->kind == Starred_kind) {
                 VISIT(c, expr, elt->v.Starred.value);
                 ADDOP_I(c, elt_loc, SET_UPDATE, depth + 1);
@@ -4829,6 +4864,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
             }
             break;
         case COMP_DICTCOMP:
+            assert(!avoid_creation);
             if (val == NULL) {
                 /* unpacking (**) case */
                 VISIT(c, expr, elt);
@@ -5002,7 +5038,7 @@ pop_inlined_comprehension_state(compiler *c, location loc,
 static int
 codegen_comprehension(compiler *c, expr_ty e, int type,
                       identifier name, asdl_comprehension_seq *generators, expr_ty elt,
-                      expr_ty val)
+                      expr_ty val, bool avoid_creation)
 {
     PyCodeObject *co = NULL;
     _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL};
@@ -5076,13 +5112,17 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
             goto error_in_scope;
         }
 
-        ADDOP_I(c, loc, op, 0);
-        if (is_inlined) {
-            ADDOP_I(c, loc, SWAP, 2);
+        if (!avoid_creation) {
+            ADDOP_I(c, loc, op, 0);
+            if (is_inlined) {
+                ADDOP_I(c, loc, SWAP, 2);
+            }
+        } else {
+            ADDOP_I(c, loc, COPY, 1);
         }
     }
     if (codegen_comprehension_generator(c, loc, generators, 0, 0,
-                                        elt, val, type, iter_state) < 0) {
+                                        elt, val, type, iter_state, avoid_creation) < 0) {
         goto error_in_scope;
     }
 
@@ -5124,6 +5164,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
         ADD_YIELD_FROM(c, loc, 1);
     }
 
+    assert(!avoid_creation);
+
     return SUCCESS;
 error_in_scope:
     if (!is_inlined) {
@@ -5145,17 +5187,17 @@ codegen_genexp(compiler *c, expr_ty e)
     _Py_DECLARE_STR(anon_genexpr, "<genexpr>");
     return codegen_comprehension(c, e, COMP_GENEXP, &_Py_STR(anon_genexpr),
                                  e->v.GeneratorExp.generators,
-                                 e->v.GeneratorExp.elt, NULL);
+                                 e->v.GeneratorExp.elt, NULL, false);
 }
 
 static int
-codegen_listcomp(compiler *c, expr_ty e)
+codegen_listcomp(compiler *c, expr_ty e, bool avoid_creation)
 {
     assert(e->kind == ListComp_kind);
     _Py_DECLARE_STR(anon_listcomp, "<listcomp>");
     return codegen_comprehension(c, e, COMP_LISTCOMP, &_Py_STR(anon_listcomp),
                                  e->v.ListComp.generators,
-                                 e->v.ListComp.elt, NULL);
+                                 e->v.ListComp.elt, NULL, avoid_creation);
 }
 
 static int
@@ -5165,7 +5207,7 @@ codegen_setcomp(compiler *c, expr_ty e)
     _Py_DECLARE_STR(anon_setcomp, "<setcomp>");
     return codegen_comprehension(c, e, COMP_SETCOMP, &_Py_STR(anon_setcomp),
                                  e->v.SetComp.generators,
-                                 e->v.SetComp.elt, NULL);
+                                 e->v.SetComp.elt, NULL, /*avoid_creation=*/false);
 }
 
 
@@ -5176,7 +5218,7 @@ codegen_dictcomp(compiler *c, expr_ty e)
     _Py_DECLARE_STR(anon_dictcomp, "<dictcomp>");
     return codegen_comprehension(c, e, COMP_DICTCOMP, &_Py_STR(anon_dictcomp),
                                  e->v.DictComp.generators,
-                                 e->v.DictComp.key, e->v.DictComp.value);
+                                 e->v.DictComp.key, e->v.DictComp.value, /*avoid_creation=*/false);
 }
 
 
@@ -5424,7 +5466,7 @@ codegen_with(compiler *c, stmt_ty s)
 }
 
 static int
-codegen_visit_expr(compiler *c, expr_ty e)
+codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused)
 {
     if (Py_EnterRecursiveCall(" during compilation")) {
         return ERROR;
@@ -5467,7 +5509,7 @@ codegen_visit_expr(compiler *c, expr_ty e)
     case GeneratorExp_kind:
         return codegen_genexp(c, e);
     case ListComp_kind:
-        return codegen_listcomp(c, e);
+        return codegen_listcomp(c, e, result_is_unused);
     case SetComp_kind:
         return codegen_setcomp(c, e);
     case DictComp_kind:
@@ -5580,6 +5622,18 @@ codegen_visit_expr(compiler *c, expr_ty e)
     return SUCCESS;
 }
 
+static int
+codegen_visit_expr(compiler *c, expr_ty e)
+{
+    return codegen_visit_expr_impl(c, e, false);
+}
+
+static int
+codegen_visit_unused_expr(compiler *c, expr_ty e)
+{
+    return codegen_visit_expr_impl(c, e, true);
+}
+
 static bool
 is_constant_slice(expr_ty s)
 {