]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-105481: remove regen-opcode. Generated _PyOpcode_Caches in regen-cases. (#108367)
authorIrit Katriel <1055913+iritkatriel@users.noreply.github.com>
Wed, 23 Aug 2023 17:39:00 +0000 (18:39 +0100)
committerGitHub <noreply@github.com>
Wed, 23 Aug 2023 17:39:00 +0000 (18:39 +0100)
19 files changed:
Include/internal/pycore_opcode.h [deleted file]
Include/internal/pycore_opcode_metadata.h
Include/internal/pycore_opcode_utils.h
Lib/opcode.py
Makefile.pre.in
Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst [new file with mode: 0644]
Objects/codeobject.c
Objects/frameobject.c
PCbuild/regen.targets
Python/assemble.c
Python/ceval.c
Python/compile.c
Python/executor.c
Python/instrumentation.c
Python/optimizer.c
Python/optimizer_analysis.c
Python/specialize.c
Tools/build/generate_opcode_h.py [deleted file]
Tools/cases_generator/generate_cases.py

diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h
deleted file mode 100644 (file)
index b47e796..0000000
+++ /dev/null
@@ -1,40 +0,0 @@
-// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py
-
-#ifndef Py_INTERNAL_OPCODE_H
-#define Py_INTERNAL_OPCODE_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE define"
-#endif
-
-#include "opcode.h"
-
-extern const uint8_t _PyOpcode_Caches[256];
-
-#ifdef NEED_OPCODE_TABLES
-
-const uint8_t _PyOpcode_Caches[256] = {
-    [LOAD_GLOBAL] = 4,
-    [BINARY_OP] = 1,
-    [UNPACK_SEQUENCE] = 1,
-    [COMPARE_OP] = 1,
-    [BINARY_SUBSCR] = 1,
-    [FOR_ITER] = 1,
-    [LOAD_SUPER_ATTR] = 1,
-    [LOAD_ATTR] = 9,
-    [STORE_ATTR] = 4,
-    [CALL] = 3,
-    [STORE_SUBSCR] = 1,
-    [SEND] = 1,
-    [JUMP_BACKWARD] = 1,
-    [TO_BOOL] = 3,
-};
-#endif   // NEED_OPCODE_TABLES
-
-#ifdef __cplusplus
-}
-#endif
-#endif  // !Py_INTERNAL_OPCODE_H
index e35db0c4c5a59d8c442c13f0b7e9779b453192f6..cc8894ad53933f5cfd1b848d25c173f886d33cd0 100644 (file)
@@ -1814,6 +1814,26 @@ const char *const _PyOpcode_OpName[268] = {
 };
 #endif // NEED_OPCODE_METADATA
 
+extern const uint8_t _PyOpcode_Caches[256];
+#ifdef NEED_OPCODE_METADATA
+const uint8_t _PyOpcode_Caches[256] = {
+    [TO_BOOL] = 3,
+    [BINARY_OP] = 1,
+    [BINARY_SUBSCR] = 1,
+    [STORE_SUBSCR] = 1,
+    [SEND] = 1,
+    [UNPACK_SEQUENCE] = 1,
+    [STORE_ATTR] = 4,
+    [LOAD_GLOBAL] = 4,
+    [LOAD_SUPER_ATTR] = 1,
+    [LOAD_ATTR] = 9,
+    [COMPARE_OP] = 1,
+    [FOR_ITER] = 1,
+    [CALL] = 3,
+    [JUMP_BACKWARD] = 1,
+};
+#endif // NEED_OPCODE_METADATA
+
 extern const uint8_t _PyOpcode_Deopt[256];
 #ifdef NEED_OPCODE_METADATA
 const uint8_t _PyOpcode_Deopt[256] = {
index f17612908cebdc03cd148b0a73a3fa61614e505b..c4acb00a4b291e34a55bf62a06fca1b908f4d384 100644 (file)
@@ -8,8 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_opcode.h"        // JUMP_FORWARD
-
+#include "opcode_ids.h"
 
 #define MAX_REAL_OPCODE 254
 
index f8487522bfdc6979eacfd4e303ce63656662e11a..386a2fba396a6a82ae7eb2ab13804ac329d8ba13 100644 (file)
@@ -5,48 +5,40 @@ operate on bytecodes (e.g. peephole optimizers).
 """
 
 
-# Note that __all__ is further extended below
-__all__ = ["cmp_op", "stack_effect", "hascompare"]
+__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap",
+           "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname",
+           "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]
 
 import _opcode
 from _opcode import stack_effect
 
-import sys
-# The build uses older versions of Python which do not have _opcode_metadata
-if sys.version_info[:2] >= (3, 13):
-    from _opcode_metadata import _specializations, _specialized_opmap
-    from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE
-    EXTENDED_ARG = opmap['EXTENDED_ARG']
+from _opcode_metadata import (_specializations, _specialized_opmap, opmap,
+                              HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE)
+EXTENDED_ARG = opmap['EXTENDED_ARG']
 
-    opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)]
-    for op, i in opmap.items():
-        opname[i] = op
-
-    __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"])
+opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)]
+for op, i in opmap.items():
+    opname[i] = op
 
 cmp_op = ('<', '<=', '==', '!=', '>', '>=')
 
-# The build uses older versions of Python which do not have _opcode.has_* functions
-if sys.version_info[:2] >= (3, 13):
-    # These lists are documented as part of the dis module's API
-    hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
-    hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
-    hasname = [op for op in opmap.values() if _opcode.has_name(op)]
-    hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
-    hasjrel = hasjump  # for backward compatibility
-    hasjabs = []
-    hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
-    haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
-    hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
+# These lists are documented as part of the dis module's API
+hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
+hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
+hasname = [op for op in opmap.values() if _opcode.has_name(op)]
+hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
+hasjrel = hasjump  # for backward compatibility
+hasjabs = []
+hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
+haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
+hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
 
-    __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel",
-                    "hasjabs", "hasfree", "haslocal", "hasexc"])
 
-    _intrinsic_1_descs = _opcode.get_intrinsic1_descs()
-    _intrinsic_2_descs = _opcode.get_intrinsic2_descs()
-    _nb_ops = _opcode.get_nb_ops()
+_intrinsic_1_descs = _opcode.get_intrinsic1_descs()
+_intrinsic_2_descs = _opcode.get_intrinsic2_descs()
+_nb_ops = _opcode.get_nb_ops()
 
-    hascompare = [opmap["COMPARE_OP"]]
+hascompare = [opmap["COMPARE_OP"]]
 
 _cache_format = {
     "LOAD_GLOBAL": {
index d66764e6165409d7641e31a1dd3c0a63bc5164c4..54b64e123cd7bcd0c460010dc81ad1193aa57796 100644 (file)
@@ -1321,7 +1321,7 @@ regen-limited-abi: all
 # Regenerate all generated files
 
 .PHONY: regen-all
-regen-all: regen-cases regen-opcode regen-typeslots \
+regen-all: regen-cases regen-typeslots \
        regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \
        regen-pegen-metaparser regen-pegen regen-test-frozenmain \
        regen-test-levenshtein regen-global-objects
@@ -1425,15 +1425,6 @@ regen-ast:
        $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast_state.h $(srcdir)/Include/internal/pycore_ast_state.h.new
        $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
 
-.PHONY: regen-opcode
-regen-opcode:
-       # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py
-       # using Tools/build/generate_opcode_h.py
-       $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \
-               $(srcdir)/Lib/opcode.py \
-               $(srcdir)/Include/internal/pycore_opcode.h.new
-       $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new
-
 .PHONY: regen-token
 regen-token:
        # Regenerate Doc/library/token-list.inc from Grammar/Tokens
@@ -1651,6 +1642,7 @@ PYTHON_HEADERS= \
                $(srcdir)/Include/object.h \
                $(srcdir)/Include/objimpl.h \
                $(srcdir)/Include/opcode.h \
+               $(srcdir)/Include/opcode_ids.h \
                $(srcdir)/Include/osdefs.h \
                $(srcdir)/Include/osmodule.h \
                $(srcdir)/Include/patchlevel.h \
@@ -1790,7 +1782,7 @@ PYTHON_HEADERS= \
                $(srcdir)/Include/internal/pycore_object_state.h \
                $(srcdir)/Include/internal/pycore_obmalloc.h \
                $(srcdir)/Include/internal/pycore_obmalloc_init.h \
-               $(srcdir)/Include/internal/pycore_opcode.h \
+               $(srcdir)/Include/internal/pycore_opcode_metadata.h \
                $(srcdir)/Include/internal/pycore_opcode_utils.h \
                $(srcdir)/Include/internal/pycore_optimizer.h \
                $(srcdir)/Include/internal/pycore_pathconfig.h \
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst
new file mode 100644 (file)
index 0000000..19746eb
--- /dev/null
@@ -0,0 +1,2 @@
+The regen-opcode build stage was removed and its work is now done in
+regen-cases.
index dca5804a91d2cde19a644a789d6c333479268454..70a0c2ebd66b2455b002c78f4214b1108987a47e 100644 (file)
@@ -6,8 +6,7 @@
 #include "pycore_code.h"          // _PyCodeConstructor
 #include "pycore_frame.h"         // FRAME_SPECIALS_SIZE
 #include "pycore_interp.h"        // PyInterpreterState.co_extra_freefuncs
-#include "pycore_opcode.h"        // _PyOpcode_Caches
-#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt
+#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_setobject.h"     // _PySet_NextEntry()
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
index 80e118e8a8aa939fe392aa9caf86d4f5b324180a..28f5a5a122280692398122829c4ce1424c945117 100644 (file)
@@ -6,8 +6,7 @@
 #include "pycore_function.h"      // _PyFunction_FromConstructor()
 #include "pycore_moduleobject.h"  // _PyModule_GetDict()
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
-#include "pycore_opcode.h"        // _PyOpcode_Caches
-#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt
+#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches
 
 
 #include "frameobject.h"          // PyFrameObject
index 2ff18a8966f55713485b4743f198ae15a2e5979b..cc9469c7ddd726b35614f01298d0ae460f1b26ec 100644 (file)
@@ -13,8 +13,6 @@
     <_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c">
       <Argument>-C</Argument>
     </_ASTOutputs>
-    <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
-    <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" />
     <_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
     <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
       <Format>rst</Format>
@@ -34,7 +32,7 @@
 
   <Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'">
     <Message Text="Touching source files to force regeneration" Importance="high" />
-    <Touch Files="@(_PegenSources);@(_ASTSources);@(_OpcodeSources);@(_TokenSources);@(_KeywordOutputs)"
+    <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs)"
            AlwaysCreate="False" />
   </Target>
 
           WorkingDirectory="$(PySourcePath)" />
   </Target>
 
-  <Target Name="_RegenOpcodes"
-          Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)"
-          DependsOnTargets="FindPythonForBuild">
-    <Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
-    <Exec Command="$(PythonForBuild) Tools\build\generate_opcode_h.py Lib\opcode.py Include\internal\pycore_opcode.h Include\internal\pycore_intrinsics.h"
-          WorkingDirectory="$(PySourcePath)" />
-  </Target>
-
   <Target Name="_RegenTokens"
           Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)"
           DependsOnTargets="FindPythonForBuild">
@@ -89,7 +79,7 @@
 
   <Target Name="Regen"
           Condition="$(Configuration) != 'PGUpdate'"
-          DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenOpcodes;_RegenTokens;_RegenKeywords;_RegenGlobalObjects">
+          DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects">
     <Message Text="Generated sources are up to date" Importance="high" />
   </Target>
 
index 4f66cf294e38c93c1b056f831c9c6d345cd2734a..c770fd108d41d66c001368b05dc950cb2eb40a93 100644 (file)
@@ -3,9 +3,8 @@
 #include "Python.h"
 #include "pycore_code.h"            // write_location_entry_start()
 #include "pycore_compile.h"
-#include "pycore_opcode.h"          // _PyOpcode_Caches[] and opcode category macros
 #include "pycore_opcode_utils.h"    // IS_BACKWARDS_JUMP_OPCODE
-#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR
+#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches
 
 
 #define DEFAULT_CODE_SIZE 128
index f7dfaebcdd8f84e44addce00d22f8b6e307cb7f5..55dfe6b1efc475619583f0e8c96ca7a0894ba928 100644 (file)
@@ -14,8 +14,7 @@
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_moduleobject.h"  // PyModuleObject
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
-#include "pycore_opcode.h"        // EXTRA_CASES
-#include "pycore_opcode_metadata.h"
+#include "pycore_opcode_metadata.h" // EXTRA_CASES
 #include "pycore_opcode_utils.h"  // MAKE_FUNCTION_*
 #include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
index b67a1885ddc983af238ec8819ca6f8876c832078..6b816b4c6eda6c0ec369977efb1bb13e58a35764 100644 (file)
@@ -24,6 +24,7 @@
 #include <stdbool.h>
 
 #include "Python.h"
+#include "opcode.h"
 #include "pycore_ast.h"           // _PyAST_GetDocString()
 #define NEED_OPCODE_TABLES
 #include "pycore_opcode_utils.h"
index 88c039da8539e2d59bfd6d5f7bee7b8c1d87ef07..0ff5106fe446ff35a559ba95f59ae258f2d82386 100644 (file)
@@ -1,5 +1,7 @@
 #include "Python.h"
 
+#include "opcode.h"
+
 #include "pycore_call.h"
 #include "pycore_ceval.h"
 #include "pycore_dict.h"
index f77c2e696f4534fc800d38fc0981f056ee5a4f90..8c7a3a06c9b93838337e2ec1ab3db2cdbda27f4d 100644 (file)
@@ -1,4 +1,7 @@
 #include "Python.h"
+
+#include "opcode_ids.h"
+
 #include "pycore_call.h"
 #include "pycore_frame.h"
 #include "pycore_interp.h"
@@ -6,8 +9,7 @@
 #include "pycore_modsupport.h"    // _PyModule_CreateInitialized()
 #include "pycore_namespace.h"
 #include "pycore_object.h"
-#include "pycore_opcode.h"
-#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE
+#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches
 #include "pycore_pyerrors.h"
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
index 57518404c3f19d7bbf28c37b4ce3f6ab3cdc034f..bbc125954c701eb04c42c4c4619d0a578875529c 100644 (file)
@@ -1,7 +1,6 @@
 #include "Python.h"
 #include "opcode.h"
 #include "pycore_interp.h"
-#include "pycore_opcode.h"
 #include "pycore_opcode_metadata.h"
 #include "pycore_opcode_utils.h"
 #include "pycore_optimizer.h"
index e48e018052c712fa2e928a80ec6f14df59b4aec1..2d177f14ff268b778537774448cfa34869c28ca9 100644 (file)
@@ -1,7 +1,6 @@
 #include "Python.h"
 #include "opcode.h"
 #include "pycore_interp.h"
-#include "pycore_opcode.h"
 #include "pycore_opcode_metadata.h"
 #include "pycore_opcode_utils.h"
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
index 2d514c0dc476d39f7be0d4c0827c1b2e4ed66f14..a467f163f2ca9d2b850b2320db7f62b6bbae528c 100644 (file)
@@ -1,4 +1,7 @@
 #include "Python.h"
+
+#include "opcode.h"
+
 #include "pycore_code.h"
 #include "pycore_descrobject.h"   // _PyMethodWrapper_Type
 #include "pycore_dict.h"
@@ -7,7 +10,7 @@
 #include "pycore_long.h"
 #include "pycore_moduleobject.h"
 #include "pycore_object.h"
-#include "pycore_opcode.h"        // _PyOpcode_Caches
+#include "pycore_opcode_metadata.h" // _PyOpcode_Caches
 #include "pycore_pylifecycle.h"   // _PyOS_URandomNonblock()
 
 
diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py
deleted file mode 100644 (file)
index 643918c..0000000
+++ /dev/null
@@ -1,67 +0,0 @@
-# This script generates the pycore_opcode.h header file.
-
-import sys
-import tokenize
-
-SCRIPT_NAME = "Tools/build/generate_opcode_h.py"
-PYTHON_OPCODE = "Lib/opcode.py"
-
-internal_header = f"""
-// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}
-
-#ifndef Py_INTERNAL_OPCODE_H
-#define Py_INTERNAL_OPCODE_H
-#ifdef __cplusplus
-extern "C" {{
-#endif
-
-#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE define"
-#endif
-
-#include "opcode.h"
-""".lstrip()
-
-internal_footer = """
-#ifdef __cplusplus
-}
-#endif
-#endif  // !Py_INTERNAL_OPCODE_H
-"""
-
-DEFINE = "#define {:<38} {:>3}\n"
-
-UINT32_MASK = (1<<32)-1
-
-def get_python_module_dict(filename):
-    mod = {}
-    with tokenize.open(filename) as fp:
-        code = fp.read()
-    exec(code, mod)
-    return mod
-
-def main(opcode_py,
-         internal_opcode_h='Include/internal/pycore_opcode.h'):
-
-    opcode = get_python_module_dict(opcode_py)
-
-    with open(internal_opcode_h, 'w') as iobj:
-        iobj.write(internal_header)
-
-        iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
-        iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
-
-        iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
-        for name, entries in opcode["_inline_cache_entries"].items():
-            iobj.write(f"    [{name}] = {entries},\n")
-        iobj.write("};\n")
-
-        iobj.write("#endif   // NEED_OPCODE_TABLES\n")
-
-        iobj.write(internal_footer)
-
-    print(f"{internal_opcode_h} regenerated from {opcode_py}")
-
-
-if __name__ == '__main__':
-    main(sys.argv[1], sys.argv[2])
index f605dcc5e4632f39c089a2c9b7ae948fc26a724c..8f9a6502e529c7c84b7336ef1891ac6c400ed71b 100644 (file)
@@ -540,6 +540,18 @@ class Generator(Analyzer):
                 for name in self.opmap:
                     self.out.emit(f'[{name}] = "{name}",')
 
+            with self.metadata_item(
+                f"const uint8_t _PyOpcode_Caches[256]",
+                "=",
+                ";",
+            ):
+                for name, _ in self.families.items():
+                    instr = self.instrs[name]
+                    if instr.cache_offset > 0:
+                        self.out.emit(f'[{name}] = {instr.cache_offset},')
+                # Irregular case:
+                self.out.emit('[JUMP_BACKWARD] = 1,')
+
             deoptcodes = {}
             for name, op in self.opmap.items():
                 if op < 256: