[3.12] GH-121970: Combine custom Pygments lexers into a package (GH-121976) (#122022)

GH-121970: Combine custom Pygments lexers into a package (GH-121976)
(cherry picked from commit 7431c3799efbd06ed03ee70b64420f45e83b3667)

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>

diff --git a/Doc/conf.py b/Doc/conf.py
index 4849d2c36e56ea030171cf58a3aed445f25de920..c12cf214e01cbb44e0c5376b291b4a32782f1edc 100644 (file)
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -18,11 +18,10 @@ from pyspecific import SOURCE_URI
 # ---------------------
 
 extensions = [
-    'asdl_highlight',
     'c_annotations',
     'escape4chm',
     'glossary_search',
-    'peg_highlight',
+    'lexers',
     'pyspecific',
     'sphinx.ext.coverage',
     'sphinx.ext.doctest',

diff --git a/Doc/tools/extensions/lexers/__init__.py b/Doc/tools/extensions/lexers/__init__.py
new file mode 100644 (file)
index 0000000..e12ac5b
--- /dev/null
+++ b/Doc/tools/extensions/lexers/__init__.py
@@ -0,0 +1,15 @@
+from .asdl_lexer import ASDLLexer
+from .peg_lexer import PEGLexer
+
+
+def setup(app):
+    # Used for highlighting Parser/Python.asdl in library/ast.rst
+    app.add_lexer("asdl", ASDLLexer)
+    # Used for highlighting Grammar/python.gram in reference/grammar.rst
+    app.add_lexer("peg", PEGLexer)
+
+    return {
+        "version": "1.0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/lexers/asdl_lexer.py
similarity index 62%
rename from Doc/tools/extensions/asdl_highlight.py
rename to Doc/tools/extensions/lexers/asdl_lexer.py
index 42863a4b3bcd6a4cb2bace948450f1158dc6b3d5..2cea058566ad85e101779aee8343e46025bc7599 100644 (file)
--- a/Doc/tools/extensions/asdl_highlight.py
+++ b/Doc/tools/extensions/lexers/asdl_lexer.py
@@ -1,15 +1,6 @@
-import sys
-from pathlib import Path
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text
 
-CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent
-sys.path.append(str(CPYTHON_ROOT / "Parser"))
-
-from pygments.lexer import RegexLexer, bygroups, include, words
-from pygments.token import (Comment, Keyword, Name, Operator,
-                            Punctuation, Text)
-
-from asdl import builtin_types
-from sphinx.highlighting import lexers
 
 class ASDLLexer(RegexLexer):
     name = "ASDL"
@@ -34,7 +25,10 @@ class ASDLLexer(RegexLexer):
                 r"(\w+)(\*\s|\?\s|\s)(\w+)",
                 bygroups(Name.Builtin.Pseudo, Operator, Name),
             ),
-            (words(builtin_types), Name.Builtin),
+            # Keep in line with ``builtin_types`` from Parser/asdl.py.
+            # ASDL's 4 builtin types are
+            # constant, identifier, int, string
+            ('constant|identifier|int|string', Name.Builtin),
             (r"attributes", Name.Builtin),
             (
                 _name + _text_ws + "(=)",
@@ -46,8 +40,3 @@ class ASDLLexer(RegexLexer):
             (r".", Text),
         ],
     }
-
-
-def setup(app):
-    lexers["asdl"] = ASDLLexer()
-    return {'version': '1.0', 'parallel_read_safe': True}

diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/lexers/peg_lexer.py
similarity index 94%
rename from Doc/tools/extensions/peg_highlight.py
rename to Doc/tools/extensions/lexers/peg_lexer.py
index 5ab5530d2699012dd733e2e09d2626d98aaec04e..06f9f8eb3121255acb776d552bce10bffb3cc324 100644 (file)
--- a/Doc/tools/extensions/peg_highlight.py
+++ b/Doc/tools/extensions/lexers/peg_lexer.py
@@ -1,8 +1,6 @@
 from pygments.lexer import RegexLexer, bygroups, include
 from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text
 
-from sphinx.highlighting import lexers
-
 
 class PEGLexer(RegexLexer):
     """Pygments Lexer for PEG grammar (.gram) files
@@ -81,8 +79,3 @@ class PEGLexer(RegexLexer):
             (r".", Text),
         ],
     }
-
-
-def setup(app):
-    lexers["peg"] = PEGLexer()
-    return {"version": "1.0", "parallel_read_safe": True}