[3.9] Include soft keywords in keyword.py (GH-20877). (GH-20880)

(cherry picked from commit 78319e373d57cd4da67660f888aa7092efbd6f24)

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>

diff --git a/Lib/keyword.py b/Lib/keyword.py
index a4db67e5597c6ef1af65762196c75d36bcb3e7f9..59fcfb0f0d177838e30725ab95baef6b2bc60927 100644 (file)
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -13,7 +13,7 @@ the python source tree and run:
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
     'False',
@@ -54,4 +54,9 @@ kwlist = [
     'yield'
 ]
 
+softkwlist = [
+
+]
+
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__

diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index ce1d6bb7bf35523cd830272515019e7f6acf7e57..58a44fbe67e8badcec2be0baa1a6bc3872c3d040 100644 (file)
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -105,6 +105,7 @@ class CCallMakerVisitor(GrammarVisitor):
         self.non_exact_tokens = non_exact_tokens
         self.cache: Dict[Any, FunctionCall] = {}
         self.keyword_cache: Dict[str, int] = {}
+        self.soft_keywords: Set[str] = set()
 
     def keyword_helper(self, keyword: str) -> FunctionCall:
         if keyword not in self.keyword_cache:
@@ -119,6 +120,7 @@ class CCallMakerVisitor(GrammarVisitor):
         )
 
     def soft_keyword_helper(self, value: str) -> FunctionCall:
+        self.soft_keywords.add(value.replace('"', ""))
         return FunctionCall(
             assigned_variable="_keyword",
             function="_PyPegen_expect_soft_keyword",

diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py
index 279c34b6dae5b9a6b5023091bfa614a9c57587e9..fa57770df4b9fd75d2124437ecd7f94b35d88160 100644 (file)
--- a/Tools/peg_generator/pegen/keywordgen.py
+++ b/Tools/peg_generator/pegen/keywordgen.py
@@ -21,13 +21,18 @@ the python source tree and run:
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
-    {keywords}
+{keywords}
+]
+
+softkwlist = [
+{soft_keywords}
 ]
 
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__
 '''.lstrip()
 
 EXTRA_KEYWORDS = ["async", "await"]
@@ -61,12 +66,12 @@ def main():
     gen.collect_todo()
 
     with args.keyword_file as thefile:
-        all_keywords = sorted(
-            list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS
-        )
+        all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS)
+        all_soft_keywords = sorted(gen.callmakervisitor.soft_keywords)
 
-        keywords = ",\n    ".join(map(repr, all_keywords))
-        thefile.write(TEMPLATE.format(keywords=keywords))
+        keywords = "" if not all_keywords else "    " + ",\n    ".join(map(repr, all_keywords))
+        soft_keywords = "" if not all_soft_keywords else "    " + ",\n    ".join(map(repr, all_soft_keywords))
+        thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords))
 
 
 if __name__ == "__main__":