gh-133541: Handle SyntaxError raised by the tokenizer on user input (#133606)

diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py
index 38cf6b5a08e89269c132b643474feccea541425b..752049ac05acdf2500a0c4ee69764f4c1a7660de 100644 (file)
--- a/Lib/_pyrepl/utils.py
+++ b/Lib/_pyrepl/utils.py
@@ -102,6 +102,8 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]:
         for color in gen_colors_from_token_stream(gen, line_lengths):
             yield color
             last_emitted = color
+    except SyntaxError:
+        return
     except tokenize.TokenError as te:
         yield from recover_unterminated_string(
             te, line_lengths, last_emitted, buffer

diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py
index 4ee320a5a4dabb6bbd2a59323cc8233bbd18454d..57526f88f9384b89a8c7096f25314443941b0e7e 100644 (file)
--- a/Lib/test/test_pyrepl/test_reader.py
+++ b/Lib/test/test_pyrepl/test_reader.py
@@ -497,6 +497,26 @@ class TestReaderInColor(ScreenEqualMixin, TestCase):
         self.assert_screen_equal(reader, code, clean=True)
         self.assert_screen_equal(reader, expected)
 
+    def test_syntax_highlighting_indentation_error(self):
+        code = dedent(
+            """\
+            def unfinished_function():
+                var = 1
+               oops
+            """
+        )
+        expected = dedent(
+            """\
+            {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z}
+                var {o}={z} {n}1{z}
+               oops
+            """
+        ).format(**colors)
+        events = code_to_events(code)
+        reader, _ = handle_all_events(events)
+        self.assert_screen_equal(reader, code, clean=True)
+        self.assert_screen_equal(reader, expected)
+
     def test_control_characters(self):
         code = 'flag = "🏳️‍🌈"'
         events = code_to_events(code)

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst
new file mode 100644 (file)
index 0000000..4f4cd84
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst
@@ -0,0 +1,2 @@
+Inconsistent indentation in user input crashed the new REPL when syntax
+highlighting was active. This is now fixed.