From: Pablo Galindo Salgado <Pablogsal@gmail.com>
Date: Mon, 5 Jun 2023 16:36:40 +0000 (+0200)
Subject: gh-105324: Fix tokenize module main function for stdin (#105325)
X-Git-Tag: v3.13.0a1~1893
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=f04c16875b649e2c2b420eb146308d0206c7e527;p=thirdparty%2FPython%2Fcpython.git

gh-105324: Fix tokenize module main function for stdin (#105325)
---

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 380dc2ab468b..a07a8bf45891 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -490,8 +490,7 @@ def main():
         else:
             filename = "<stdin>"
             tokens = _generate_tokens_from_c_tokenizer(
-                (x.encode('utf-8') for x in iter(sys.stdin.readline, "")
-            ), "utf-8", extra_tokens=True)
+                sys.stdin.readline, extra_tokens=True)
 
 
         # Output the tokenization
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst
new file mode 100644
index 000000000000..17275aed338d
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst	
@@ -0,0 +1,2 @@
+Fix the main function of the :mod:`tokenize` module when reading from
+``sys.stdin``. Patch by Pablo Galindo