]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
[3.12] gh-105324: Fix tokenize module main function for stdin (GH-105325) (#105330)
authorMiss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
Mon, 5 Jun 2023 19:44:20 +0000 (12:44 -0700)
committerGitHub <noreply@github.com>
Mon, 5 Jun 2023 19:44:20 +0000 (21:44 +0200)
Lib/tokenize.py
Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst [new file with mode: 0644]

index 380dc2ab468b57ac677a36de0c79bc2b86e5f3e3..a07a8bf45891ac6d9fd682875cbc1f3c1d38437a 100644 (file)
@@ -490,8 +490,7 @@ def main():
         else:
             filename = "<stdin>"
             tokens = _generate_tokens_from_c_tokenizer(
-                (x.encode('utf-8') for x in iter(sys.stdin.readline, "")
-            ), "utf-8", extra_tokens=True)
+                sys.stdin.readline, extra_tokens=True)
 
 
         # Output the tokenization
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst
new file mode 100644 (file)
index 0000000..17275ae
--- /dev/null
@@ -0,0 +1,2 @@
+Fix the main function of the :mod:`tokenize` module when reading from
+``sys.stdin``. Patch by Pablo Galindo