bpo-45548: Some test modules must be built as shared libs (GH-29268)

Some test cases don't work when test modules are static extensions.

Add dependency on Modules/config.c to trigger a rebuild whenever a
module build type is changed.

``makesetup`` puts shared extensions into ``Modules/`` directory. Create
symlinks from pybuilddir so the extensions can be imported.

Note: It is not possible to use the content of pybuilddir.txt as a build
target. Makefile evaluates target variables in the first pass. The
pybuilddir.txt file does not exist at that point.

diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 21ad4669476ef26c83bb42c9f7256b2bb233eae5..156bfbd9109ee89e1d3995e9469b7388a2927cd5 100644 (file)
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -507,6 +507,13 @@ Build Changes
   except empty tuple singleton.
   (Contributed by Christian Heimes in :issue:`45522`)
 
+* ``Modules/Setup`` and ``Modules/makesetup`` have been improved and tied up.
+  Extension modules can now be built through ``makesetup``. All except some
+  test modules can be linked statically into main binary or library.
+  (Contributed by Brett Cannon and Christian Heimes in :issue:`45548`,
+  :issue:`45570`, :issue:`45571`, and :issue:`43974`.)
+
+
 C API Changes
 =============
 

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 60acc16beeee3f90bc8a1460bb3fd5b1f5b90f23..322800ae26e4ca35151dedf60cc432d7178f1fc2 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -705,8 +705,17 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
        fi
 
 
-oldsharedmods: $(SHAREDMODS)
-
+# create relative links from build/lib.platform/egg.so to Modules/egg.so
+# pybuilddir.txt is created too late. We cannot use it in Makefile
+# targets. ln --relative is not portable.
+oldsharedmods: $(SHAREDMODS) pybuilddir.txt
+       @target=`cat pybuilddir.txt`; \
+       $(MKDIR_P) $$target; \
+       for mod in X $(SHAREDMODS); do \
+               if test $$mod != X; then \
+                       $(LN) -sf ../../$$mod $$target/`basename $$mod`; \
+               fi; \
+       done
 
 Makefile Modules/config.c: Makefile.pre \
                                $(srcdir)/Modules/config.c.in \

diff --git a/Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst b/Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst
new file mode 100644 (file)
index 0000000..e6ccd52
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst
@@ -0,0 +1,4 @@
+``Modules/Setup`` and ``Modules/makesetup`` have been improved. The
+``Setup`` file now contains working rules for all extensions. Outdated
+comments have been removed. Rules defined by ``makesetup`` track
+dependencies correctly.

diff --git a/Modules/Setup b/Modules/Setup
index b4eae38fdff60c04b03f1f3e8388ea7552b7400a..57584f38fd8e701aa7e28a91cebd7bded75e65e3 100644 (file)
--- a/Modules/Setup
+++ b/Modules/Setup
@@ -297,14 +297,16 @@ xxsubtype xxsubtype.c  # Required for the test suite to pass!
 
 #_xxsubinterpreters _xxsubinterpretersmodule.c
 #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
-#_ctypes_test _ctypes/_ctypes_test.c
 #_testbuffer _testbuffer.c
-#_testimportmultiple _testimportmultiple.c
 #_testinternalcapi _testinternalcapi.c
-#_testmultiphase _testmultiphase.c
+
+# Some testing modules MUST be built as shared libraries.
 
 #*shared*
-#_testcapi _testcapimodule.c  # CANNOT be statically compiled!
+#_ctypes_test _ctypes/_ctypes_test.c
+#_testcapi _testcapimodule.c
+#_testimportmultiple _testimportmultiple.c
+#_testmultiphase _testmultiphase.c
 
 # ---
 # Uncommenting the following line tells makesetup that all following modules

diff --git a/Modules/makesetup b/Modules/makesetup
index 7547315281643572b8f975bd8d2bf131dbcb639a..543992c4a02941d4b55a26a14eaa00bbef1b938d 100755 (executable)
--- a/Modules/makesetup
+++ b/Modules/makesetup
@@ -241,7 +241,8 @@ sed -e 's/[         ]*#.*//' -e '/^[        ]*$/d' |
                                cc="$cc \$(PY_BUILTIN_MODULE_CFLAGS)";;
                        esac
                        mods_upper=$(echo $mods | tr '[a-z]' '[A-Z]')
-                       rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS); $cc $cpps -c $src -o $obj"
+                       # force rebuild when header file or module build flavor (static/shared) is changed
+                       rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS) Modules/config.c; $cc $cpps -c $src -o $obj"
                        echo "$rule" >>$rulesf
                done
                case $doconfig in

diff --git a/setup.py b/setup.py
index f32dd4c6e5e14552c11811dd91f426e94f598aa4..6fa8a674eda31047c277434db8633cdcf2b92a7e 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -426,12 +426,13 @@ class PyBuildExt(build_ext):
             # re-compile extensions if a header file has been changed
             ext.depends.extend(headers)
 
-    def remove_configured_extensions(self):
+    def handle_configured_extensions(self):
         # The sysconfig variables built by makesetup that list the already
         # built modules and the disabled modules as configured by the Setup
         # files.
-        sysconf_built = sysconfig.get_config_var('MODBUILT_NAMES').split()
-        sysconf_dis = sysconfig.get_config_var('MODDISABLED_NAMES').split()
+        sysconf_built = set(sysconfig.get_config_var('MODBUILT_NAMES').split())
+        sysconf_shared = set(sysconfig.get_config_var('MODSHARED_NAMES').split())
+        sysconf_dis = set(sysconfig.get_config_var('MODDISABLED_NAMES').split())
 
         mods_built = []
         mods_disabled = []
@@ -449,11 +450,15 @@ class PyBuildExt(build_ext):
                                mods_configured]
             # Remove the shared libraries built by a previous build.
             for ext in mods_configured:
+                # Don't remove shared extensions which have been built
+                # by Modules/Setup
+                if ext.name in sysconf_shared:
+                    continue
                 fullpath = self.get_ext_fullpath(ext.name)
-                if os.path.exists(fullpath):
+                if os.path.lexists(fullpath):
                     os.unlink(fullpath)
 
-        return (mods_built, mods_disabled)
+        return mods_built, mods_disabled
 
     def set_compiler_executables(self):
         # When you run "make CC=altcc" or something similar, you really want
@@ -478,7 +483,7 @@ class PyBuildExt(build_ext):
             self.remove_disabled()
 
         self.update_sources_depends()
-        mods_built, mods_disabled = self.remove_configured_extensions()
+        mods_built, mods_disabled = self.handle_configured_extensions()
         self.set_compiler_executables()
 
         if LIST_MODULE_NAMES: