Include soft keywords in keyword.py (GH-20877)

diff --git a/Lib/keyword.py b/Lib/keyword.py
index b6a99825702117d1063334d9bd08bbcc615f88e4..ccc951500f6d8c0f74d7d133116bd68daf87f2a1 100644 (file)
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -13,7 +13,7 @@ the python source tree and run:
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
     'False',
@@ -53,4 +53,9 @@ kwlist = [
     'yield'
 ]
 
+softkwlist = [
+
+]
+
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__

diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index ce1d6bb7bf35523cd830272515019e7f6acf7e57..58a44fbe67e8badcec2be0baa1a6bc3872c3d040 100644 (file)
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -105,6 +105,7 @@ class CCallMakerVisitor(GrammarVisitor):
         self.non_exact_tokens = non_exact_tokens
         self.cache: Dict[Any, FunctionCall] = {}
         self.keyword_cache: Dict[str, int] = {}
+        self.soft_keywords: Set[str] = set()
 
     def keyword_helper(self, keyword: str) -> FunctionCall:
         if keyword not in self.keyword_cache:
@@ -119,6 +120,7 @@ class CCallMakerVisitor(GrammarVisitor):
         )
 
     def soft_keyword_helper(self, value: str) -> FunctionCall:
+        self.soft_keywords.add(value.replace('"', ""))
         return FunctionCall(
             assigned_variable="_keyword",
             function="_PyPegen_expect_soft_keyword",

diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py
index 86849440966544d498cc43e36190beab9698f500..639f01bf2373e6a24afd5453e81cb93c778a7207 100644 (file)
--- a/Tools/peg_generator/pegen/keywordgen.py
+++ b/Tools/peg_generator/pegen/keywordgen.py
@@ -21,13 +21,18 @@ the python source tree and run:
 Alternatively, you can run 'make regen-keyword'.
 """
 
-__all__ = ["iskeyword", "kwlist"]
+__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
 
 kwlist = [
-    {keywords}
+{keywords}
+]
+
+softkwlist = [
+{soft_keywords}
 ]
 
 iskeyword = frozenset(kwlist).__contains__
+issoftkeyword = frozenset(softkwlist).__contains__
 '''.lstrip()
 
 EXTRA_KEYWORDS = ["async", "await"]
@@ -60,9 +65,11 @@ def main():
 
     with args.keyword_file as thefile:
         all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS)
+        all_soft_keywords = sorted(gen.callmakervisitor.soft_keywords)
 
-        keywords = ",\n    ".join(map(repr, all_keywords))
-        thefile.write(TEMPLATE.format(keywords=keywords))
+        keywords = "    " + ",\n    ".join(map(repr, all_keywords))
+        soft_keywords = "    " + ",\n    ".join(map(repr, all_soft_keywords))
+        thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords))
 
 
 if __name__ == "__main__":