gh-108740: Fix "make regen-all" race condition (#108741)

Fix a race condition in "make regen-all". The deepfreeze.c source and
files generated by Argument Clinic are now generated or updated
before generating "global objects". Previously, some identifiers may
miss depending on the order in which these files were generated.

* "make regen-global-objects": Make sure that deepfreeze.c is
  generated and up to date, and always run "make clinic".
* "make clinic" no longer runs generate_global_objects.py script.
* "make regen-deepfreeze" now only updates deepfreeze.c (C file).
  It doesn't build deepfreeze.o (object) anymore.
* Remove misleading messages in "make regen-global-objects" and
  "make clinic". They are now outdated, these commands are now
  safe to use.
* Document generates files in Doc/using/configure.rst.

Co-authored-by: Erlend E. Aasland <erlend@python.org>

diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst
index ad58255c33b33b87a72e5b1913d82347bb25778b..ba1898e35b5fbc267e6837be4bf016334dc28220 100644 (file)
--- a/Doc/using/configure.rst
+++ b/Doc/using/configure.rst
@@ -60,6 +60,29 @@ See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform
 support".
 
 
+Generated files
+===============
+
+To reduce build dependencies, Python source code contains multiple generated
+files. Commands to regenerate all generated files::
+
+    make regen-all
+    make regen-stdlib-module-names
+    make regen-limited-abi
+    make regen-configure
+
+The ``Makefile.pre.in`` file documents generated files, their inputs, and tools used
+to regenerate them. Search for ``regen-*`` make targets.
+
+The ``make regen-configure`` command runs `tiran/cpython_autoconf
+<https://github.com/tiran/cpython_autoconf>`_ container for reproducible build;
+see container ``entry.sh`` script. The container is optional, the following
+command can be run locally, the generated files depend on autoconf and aclocal
+versions::
+
+    autoreconf -ivf -Werror
+
+
 .. _configure-options:
 
 Configure Options

diff --git a/Makefile.pre.in b/Makefile.pre.in
index cf77cbd13ad4d94b2ddb3ba8ff269847362efb05..aa3eaabc7559f61a454b103d33f467cd46c7b915 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -490,6 +490,7 @@ OBJECT_OBJS=        \
                Objects/weakrefobject.o \
                @PERF_TRAMPOLINE_OBJ@
 
+DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c
 DEEPFREEZE_OBJS = Python/deepfreeze/deepfreeze.o
 
 ##########################################################################
@@ -777,7 +778,6 @@ coverage-report: regen-token regen-frozen
 .PHONY: clinic
 clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
        $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir)
-       $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py
 
 .PHONY: clinic-tests
 clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c
@@ -1252,12 +1252,12 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN)
 # Deepfreeze targets
 
 .PHONY: regen-deepfreeze
-regen-deepfreeze: $(DEEPFREEZE_OBJS)
+regen-deepfreeze: $(DEEPFREEZE_C)
 
 DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT)
 
 # BEGIN: deepfreeze modules
-Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)
+$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)
        $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \
        Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \
        Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \
@@ -1284,8 +1284,6 @@ Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)
        Python/frozen_modules/frozen_only.h:frozen_only \
        -o Python/deepfreeze/deepfreeze.c
 # END: deepfreeze modules
-       @echo "Note: Deepfreeze may have added some global objects,"
-       @echo "      so run 'make regen-global-objects' if necessary."
 
 # We keep this renamed target around for folks with muscle memory.
 .PHONY: regen-importlib
@@ -1294,11 +1292,12 @@ regen-importlib: regen-frozen
 ############################################################################
 # Global objects
 
+# Dependencies which can add and/or remove _Py_ID() identifiers:
+# - deepfreeze.c
+# - "make clinic"
 .PHONY: regen-global-objects
-regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py
+regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py $(DEEPFREEZE_C) clinic
        $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py
-       @echo "Note: Global objects can be added or removed by other tools (e.g. deepfreeze), "
-       @echo "      so be sure to re-run regen-global-objects after those tools."
 
 ############################################################################
 # ABI
@@ -1320,9 +1319,10 @@ regen-limited-abi: all
 ############################################################################
 # Regenerate all generated files
 
+# "clinic" is regenerated implicitly via "regen-global-objects".
 .PHONY: regen-all
 regen-all: regen-cases regen-typeslots \
-       regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \
+       regen-token regen-ast regen-keyword regen-sre regen-frozen \
        regen-pegen-metaparser regen-pegen regen-test-frozenmain \
        regen-test-levenshtein regen-global-objects
        @echo
@@ -2597,6 +2597,7 @@ recheck:
 autoconf:
        (cd $(srcdir); autoreconf -ivf -Werror)
 
+# See https://github.com/tiran/cpython_autoconf container
 .PHONY: regen-configure
 regen-configure:
        @if command -v podman >/dev/null; then RUNTIME="podman"; else RUNTIME="docker"; fi; \

diff --git a/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst b/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst
new file mode 100644 (file)
index 0000000..190d503
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst
@@ -0,0 +1,4 @@
+Fix a race condition in ``make regen-all``. The ``deepfreeze.c`` source and
+files generated by Argument Clinic are now generated or updated before
+generating "global objects". Previously, some identifiers may miss depending
+on the order in which these files were generated. Patch by Victor Stinner.

diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py
index ee4dd2f8682ba41e1650758d45f4a8d7f1d005c4..12200979fa4bc76b0215ad562412e222090fb757 100644 (file)
--- a/Tools/build/freeze_modules.py
+++ b/Tools/build/freeze_modules.py
@@ -585,7 +585,7 @@ def regen_makefile(modules):
     pyfiles = []
     frozenfiles = []
     rules = ['']
-    deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)",
+    deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)",
                        "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"]
     for src in _iter_sources(modules):
         frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)