[3.11] gh-108740: Fix "make regen-all" race condition (#108741) (#109021)

gh-108740: Fix "make regen-all" race condition (#108741)

Fix a race condition in "make regen-all". The deepfreeze.c source and
files generated by Argument Clinic are now generated or updated
before generating "global objects". Previously, some identifiers may
miss depending on the order in which these files were generated.

* "make regen-global-objects": Make sure that deepfreeze.c is
  generated and up to date, and always run "make clinic".
* "make regen-deepfreeze" now only updates deepfreeze.c (C file).
  It doesn't build deepfreeze.o (object) anymore.
* Remove misleading messages in "make regen-global-objects" and
  "make clinic". They are now outdated, these commands are now
  safe to use.

Backport notes:

* Omit Doc/using/configure.rst changes.
* no need to change "make clinic", it didn't run
  generate_global_objects.py script before.

Co-authored-by: Erlend E. Aasland <erlend@python.org>
(cherry picked from commit db1ee6a19ab62191c16ecb732cb4dcaede98a902)

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 884bea1841c232d5a79e81263b0f7d923cbfea2e..99b1be2206c7234966d0f13ce9febced512ecece 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -487,6 +487,7 @@ OBJECT_OBJS=        \
                Objects/unionobject.o \
                Objects/weakrefobject.o
 
+DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c
 DEEPFREEZE_OBJS = Python/deepfreeze/deepfreeze.o
 
 ##########################################################################
@@ -1152,12 +1153,12 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
 # Deepfreeze targets
 
 .PHONY: regen-deepfreeze
-regen-deepfreeze: $(DEEPFREEZE_OBJS)
+regen-deepfreeze: $(DEEPFREEZE_C)
 
 DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT)
 
 # BEGIN: deepfreeze modules
-Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)
+$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)
        $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \
        Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \
        Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \
@@ -1184,8 +1185,6 @@ Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)
        Python/frozen_modules/frozen_only.h:frozen_only \
        -o Python/deepfreeze/deepfreeze.c
 # END: deepfreeze modules
-       @echo "Note: Deepfreeze may have added some global objects,"
-       @echo "      so run 'make regen-global-objects' if necessary."
 
 # We keep this renamed target around for folks with muscle memory.
 .PHONY: regen-importlib
@@ -1194,11 +1193,12 @@ regen-importlib: regen-frozen
 ############################################################################
 # Global objects
 
+# Dependencies which can add and/or remove _Py_ID() identifiers:
+# - deepfreeze.c
+# - "make clinic"
 .PHONY: regen-global-objects
-regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py
+regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py $(DEEPFREEZE_C) clinic
        $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
-       @echo "Note: Global objects can be added or removed by other tools (e.g. deepfreeze), "
-       @echo "      so be sure to re-run regen-global-objects after those tools."
 
 ############################################################################
 # ABI
@@ -1217,8 +1217,10 @@ regen-limited-abi: all
 ############################################################################
 # Regenerate all generated files
 
+# "clinic" is regenerated implicitly via "regen-global-objects".
+.PHONY: regen-all
 regen-all: regen-opcode regen-opcode-targets regen-typeslots \
-       regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \
+       regen-token regen-ast regen-keyword regen-sre regen-frozen \
        regen-pegen-metaparser regen-pegen regen-test-frozenmain \
        regen-global-objects
        @echo
@@ -2389,6 +2391,7 @@ recheck:
 autoconf:
        (cd $(srcdir); autoreconf -ivf -Werror)
 
+# See https://github.com/tiran/cpython_autoconf container
 .PHONY: regen-configure
 regen-configure:
        @if command -v podman >/dev/null; then RUNTIME="podman"; else RUNTIME="docker"; fi; \

diff --git a/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst b/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst
new file mode 100644 (file)
index 0000000..190d503
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst
@@ -0,0 +1,4 @@
+Fix a race condition in ``make regen-all``. The ``deepfreeze.c`` source and
+files generated by Argument Clinic are now generated or updated before
+generating "global objects". Previously, some identifiers may miss depending
+on the order in which these files were generated. Patch by Victor Stinner.

diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index dd208c7847194381092606587f1a66c3ae2c3cdb..0e41d96d4623a8102b17b481320b3af25d5109ea 100644 (file)
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -581,7 +581,7 @@ def regen_makefile(modules):
     pyfiles = []
     frozenfiles = []
     rules = ['']
-    deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)",
+    deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)",
                        "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \\"]
     for src in _iter_sources(modules):
         frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)