bpo-46712: Do not Regen Deep-Frozen Modules before Generating Global Objects (gh...

We have to run "make regen-deepfreeze" before running Tools/scripts/generate-global-objects.py; otherwise we will miss any changes to global objects in deep-frozen modules (which aren't committed in the repo).  However, building $(PYTHON_FOR_FREEZE) fails if one of its source files had a global object (e.g. via _Py_ID(...)) added or removed, without generate-global-objects.py running first.  So "make regen-global-objects" would sometimes fail.

We solve this by running generate-global-objects.py before *and* after "make regen-deepfreeze". To speed things up and cut down on noise, we also avoid updating the global objects files if there are no changes to them.

https://bugs.python.org/issue46712

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 4b9aab0a284ced217357eb5aa3ec81fe530ecb87..8eab27ccf9dc37bcd89f181bceb5d047bbf026f5 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1179,7 +1179,12 @@ regen-importlib: regen-frozen
 # Global objects
 
 .PHONY: regen-global-objects
-regen-global-objects: regen-deepfreeze $(srcdir)/Tools/scripts/generate_global_objects.py
+regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py
+       $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
+       @# Run one more time after deepfreezing, to catch any globals added
+       @# there.  This is necessary because the deep-frozen code isn't
+       @# commited to the repo.
+       $(MAKE) regen-deepfreeze
        $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
 
 ############################################################################

diff --git a/Tools/scripts/generate_global_objects.py b/Tools/scripts/generate_global_objects.py
index 17ddb8b324a0b62928fb5260705ddc7f6f8ed0e9..f7653604e822b21297909329f0ed462f32e0edc9 100644 (file)
--- a/Tools/scripts/generate_global_objects.py
+++ b/Tools/scripts/generate_global_objects.py
@@ -1,5 +1,6 @@
 import contextlib
 import glob
+import io
 import os.path
 import re
 import sys
@@ -123,6 +124,7 @@ def iter_global_strings():
                     varname, string = m.groups()
                     yield varname, string, filename, lno, line
 
+
 def iter_to_marker(lines, marker):
     for line in lines:
         if line.rstrip() == marker:
@@ -165,6 +167,19 @@ class Printer:
         self.write("}" + suffix)
 
 
+@contextlib.contextmanager
+def open_for_changes(filename, orig):
+    """Like open() but only write to the file if it changed."""
+    outfile = io.StringIO()
+    yield outfile
+    text = outfile.getvalue()
+    if text != orig:
+        with open(filename, 'w', encoding='utf-8') as outfile:
+            outfile.write(text)
+    else:
+        print(f'# not changed: {filename}')
+
+
 #######################################
 # the global objects
 
@@ -177,13 +192,15 @@ def generate_global_strings(identifiers, strings):
 
     # Read the non-generated part of the file.
     with open(filename) as infile:
-        before = ''.join(iter_to_marker(infile, START))[:-1]
-        for _ in iter_to_marker(infile, END):
-            pass
-        after = infile.read()[:-1]
+        orig = infile.read()
+    lines = iter(orig.rstrip().splitlines())
+    before = '\n'.join(iter_to_marker(lines, START))
+    for _ in iter_to_marker(lines, END):
+        pass
+    after = '\n'.join(lines)
 
     # Generate the file.
-    with open(filename, 'w', encoding='utf-8') as outfile:
+    with open_for_changes(filename, orig) as outfile:
         printer = Printer(outfile)
         printer.write(before)
         printer.write(START)
@@ -202,7 +219,6 @@ def generate_global_strings(identifiers, strings):
             with printer.block('struct', ' latin1[128];'):
                 printer.write("PyCompactUnicodeObject _latin1;")
                 printer.write("uint8_t _data[2];")
-
         printer.write(END)
         printer.write(after)
 
@@ -227,13 +243,15 @@ def generate_runtime_init(identifiers, strings):
 
     # Read the non-generated part of the file.
     with open(filename) as infile:
-        before = ''.join(iter_to_marker(infile, START))[:-1]
-        for _ in iter_to_marker(infile, END):
-            pass
-        after = infile.read()[:-1]
+        orig = infile.read()
+    lines = iter(orig.rstrip().splitlines())
+    before = '\n'.join(iter_to_marker(lines, START))
+    for _ in iter_to_marker(lines, END):
+        pass
+    after = '\n'.join(lines)
 
     # Generate the file.
-    with open(filename, 'w', encoding='utf-8') as outfile:
+    with open_for_changes(filename, orig) as outfile:
         printer = Printer(outfile)
         printer.write(before)
         printer.write(START)
@@ -286,6 +304,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]':
                 raise ValueError(f'string mismatch for {name!r} ({string!r} != {strings[name]!r}')
     return identifiers, strings
 
+
 #######################################
 # the script