From: Steve Dower Date: Wed, 15 Sep 2021 17:11:12 +0000 (+0100) Subject: bpo-45188: Windows now regenerates frozen modules at the start of build instead of... X-Git-Tag: v3.11.0a1~172 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=09b4ad11f323f8702cde795e345b75e0fbb1a9a5;p=thirdparty%2FPython%2Fcpython.git bpo-45188: Windows now regenerates frozen modules at the start of build instead of late (GH-28322) This will enable us to drop the frozen module header files from the repository. It does currently cause many source files to be built twice, which just takes more time. For whoever comes to fix this in the future, the files shared between freeze_module and pythoncore should be put into a static library that is consumed by both. --- diff --git a/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst b/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst new file mode 100644 index 000000000000..df470e8eeb30 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst @@ -0,0 +1,3 @@ +Windows builds now regenerate frozen modules as the first part of the build. +Previously the regeneration was later in the build, which would require it +to be restarted if any modules had changed. diff --git a/PC/config_minimal.c b/PC/config_minimal.c new file mode 100644 index 000000000000..adb1c44a7242 --- /dev/null +++ b/PC/config_minimal.c @@ -0,0 +1,53 @@ +/* Module configuration */ + +/* This file contains the table of built-in modules. + See create_builtin() in import.c. */ + +#include "Python.h" + +extern PyObject* PyInit_faulthandler(void); +extern PyObject* PyInit__tracemalloc(void); +extern PyObject* PyInit_gc(void); +extern PyObject* PyInit_nt(void); +extern PyObject* PyInit__signal(void); +extern PyObject* PyInit_winreg(void); + +extern PyObject* PyInit__ast(void); +extern PyObject* PyInit__io(void); +extern PyObject* PyInit_atexit(void); +extern PyObject* _PyWarnings_Init(void); +extern PyObject* PyInit__string(void); +extern PyObject* PyInit__tokenize(void); + +extern PyObject* PyMarshal_Init(void); +extern PyObject* PyInit__imp(void); + +struct _inittab _PyImport_Inittab[] = { + {"_ast", PyInit__ast}, + {"faulthandler", PyInit_faulthandler}, + {"gc", PyInit_gc}, + {"nt", PyInit_nt}, /* Use the NT os functions, not posix */ + {"_signal", PyInit__signal}, + {"_tokenize", PyInit__tokenize}, + {"_tracemalloc", PyInit__tracemalloc}, + + {"winreg", PyInit_winreg}, + + /* This module "lives in" with marshal.c */ + {"marshal", PyMarshal_Init}, + + /* This lives it with import.c */ + {"_imp", PyInit__imp}, + + /* These entries are here for sys.builtin_module_names */ + {"builtins", NULL}, + {"sys", NULL}, + {"_warnings", _PyWarnings_Init}, + {"_string", PyInit__string}, + + {"_io", PyInit__io}, + {"atexit", PyInit_atexit}, + + /* Sentinel */ + {0, 0} +}; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a0bedf49e699..a87dca75c34f 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -88,24 +88,145 @@ - _CONSOLE;%(PreprocessorDefinitions) + Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) Console + version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) + - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - true - true - false - true - false - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -126,7 +247,7 @@ hello - $(IntDir)ello.g.h + $(IntDir)hello.g.h $(PySourcePath)Python\frozen_modules\hello.h @@ -134,9 +255,9 @@ - + - + @@ -145,18 +266,8 @@ - - - - - - - + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index f464ad3b18e4..b3cbd471c66a 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -14,6 +14,18 @@ + + $(Platform) + Win32 + x64 + $(Configuration) + Release + + Build + Clean + CleanAll + false + $(Platform) $(Configuration) @@ -73,7 +85,7 @@ - + @@ -82,6 +94,11 @@ + ' else: - source = os.path.relpath(self.pyfile, ROOT_DIR) + source = relpath_for_posix_display(self.pyfile, ROOT_DIR) return { 'module': self.name, 'ispkg': self.ispkg, @@ -397,7 +432,7 @@ def replace_block(lines, start_marker, end_marker, replacements, file): raise Exception(f"End marker {end_marker!r} " f"occurs before start marker {start_marker!r} " f"in file {file}") - replacements = [line.rstrip() + os.linesep for line in replacements] + replacements = [line.rstrip() + '\n' for line in replacements] return lines[:start_pos + 1] + replacements + lines[end_pos:] @@ -446,7 +481,7 @@ def regen_frozen(modules): for src in _iter_sources(modules): # Adding a comment to separate sections here doesn't add much, # so we don't. - header = os.path.relpath(src.frozenfile, parentdir) + header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') deflines = [] @@ -503,11 +538,10 @@ def regen_makefile(modules): frozenfiles = [] rules = [''] for src in _iter_sources(modules): - header = os.path.relpath(src.frozenfile, ROOT_DIR) - relfile = header.replace('\\', '/') - frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') + header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) + frozenfiles.append(f'\t\t$(srcdir)/{header} \\') - pyfile = os.path.relpath(src.pyfile, ROOT_DIR) + pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) # Note that we freeze the module to the target .h file # instead of going through an intermediate file like we used to. rules.append(f'{header}: Programs/_freeze_module {pyfile}') @@ -546,9 +580,9 @@ def regen_pcbuild(modules): # See bpo-45186 and bpo-45188. if src.id not in ESSENTIAL and src.id != 'hello': continue - pyfile = os.path.relpath(src.pyfile, ROOT_DIR).replace('/', '\\') - header = os.path.relpath(src.frozenfile, ROOT_DIR).replace('/', '\\') - intfile = header.split('\\')[-1].strip('.h') + '.g.h' + pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) + header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) + intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') @@ -600,7 +634,7 @@ def _freeze_module(frozenid, pyfile, frozenfile): print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True) try: subprocess.run(argv, check=True) - except subprocess.CalledProcessError: + except (FileNotFoundError, subprocess.CalledProcessError): if not os.path.exists(TOOL): sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"') raise # re-raise