bpo-44133: Link Python executable with object files (GH-30556)

When Python is built without --enable-shared, the "python" program is
now linked to object files, rather than being linked to the Python
library (libpython.a), to make sure that all symbols are exported.
Previously, the linker omitted some symbols like the Py_FrozenMain()
function.

When Python is configured with --without-static-libpython, the Python
static library (libpython.a) is no longer built.

* Check --without-static-libpython earlier in configure.ac
* Add LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variables to Makefile.
* test_capi now ensures that the "Py_FrozenMain" symbol is exported.

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 99263bff091bebe88ee86d0c0508455cc47e016e..9f217852ec529f36946cc64ad368f872ef288639 100644 (file)
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -643,6 +643,24 @@ class CAPITest(unittest.TestCase):
         expected = compile(code, "<string>", "exec")
         self.assertEqual(result.co_consts, expected.co_consts)
 
+    def test_export_symbols(self):
+        # bpo-44133: Ensure that the "Py_FrozenMain" and
+        # "PyThread_get_thread_native_id" symbols are exported by the Python
+        # (directly by the binary, or via by the Python dynamic library).
+        ctypes = import_helper.import_module('ctypes')
+        names = ['PyThread_get_thread_native_id']
+
+        # Python/frozenmain.c fails to build on Windows when the symbols are
+        # missing:
+        # - PyWinFreeze_ExeInit
+        # - PyWinFreeze_ExeTerm
+        # - PyInitFrozenExtensions
+        if os.name != 'nt':
+            names.append('Py_FrozenMain')
+        for name in names:
+            with self.subTest(name=name):
+                self.assertTrue(hasattr(ctypes.pythonapi, name))
+
 
 class TestPendingCalls(unittest.TestCase):
 

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 41b123abcef11d859bf564c9323df35a780857fd..a84badcd4938903b22827c6fa4578ebb6372a050 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -265,6 +265,7 @@ DLLLIBRARY= @DLLLIBRARY@
 LDLIBRARYDIR=   @LDLIBRARYDIR@
 INSTSONAME=    @INSTSONAME@
 LIBRARY_DEPS=  @LIBRARY_DEPS@
+LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@
 PY_ENABLE_SHARED=      @PY_ENABLE_SHARED@
 STATIC_LIBPYTHON=      @STATIC_LIBPYTHON@
 
@@ -526,6 +527,8 @@ LIBRARY_OBJS=       \
                Modules/getpath.o \
                Python/frozen.o
 
+LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
+
 ##########################################################################
 # DTrace
 
@@ -721,8 +724,8 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
        $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir)
 
 # Build the interpreter
-$(BUILDPYTHON):        Programs/python.o $(LIBRARY_DEPS)
-       $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
+$(BUILDPYTHON):        Programs/python.o $(LINK_PYTHON_DEPS)
+       $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 platform: $(BUILDPYTHON) pybuilddir.txt
        $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
@@ -965,8 +968,8 @@ regen-test-frozenmain: $(BUILDPYTHON)
        # using Programs/freeze_test_frozenmain.py
        $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Programs/freeze_test_frozenmain.py Programs/test_frozenmain.h
 
-Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
-       $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
+Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS)
+       $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 ############################################################################
 # "Bootstrap Python" used to run deepfreeze.py

diff --git a/Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst b/Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst
new file mode 100644 (file)
index 0000000..7c2a48a
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst
@@ -0,0 +1,5 @@
+When Python is built without :option:`--enable-shared`, the ``python``
+program is now linked to object files, rather than being linked to the Python
+static library (libpython.a), to make sure that all symbols are exported.
+Previously, the linker omitted some symbols like the :c:func:`Py_FrozenMain`
+function. Patch by Victor Stinner.

diff --git a/Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst b/Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst
new file mode 100644 (file)
index 0000000..3542850
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst
@@ -0,0 +1,2 @@
+When Python is configured with :option:`--without-static-libpython`, the Python
+static library (libpython.a) is no longer built. Patch by Victor Stinner.

diff --git a/configure b/configure
index 127b350b4bb04503bf2ef11375483737eab9e97e..ccbf4fc0b180f7d64b8cf35fbb8b253d0486c8fc 100755 (executable)
--- a/configure
+++ b/configure
@@ -775,8 +775,6 @@ MODULE__IO_TRUE
 MODULES_SETUP_STDLIB
 MODULE_BUILDTYPE
 TEST_MODULES
-LIBRARY_DEPS
-STATIC_LIBPYTHON
 OPENSSL_RPATH
 OPENSSL_LDFLAGS
 OPENSSL_LIBS
@@ -877,6 +875,10 @@ READELF
 ARFLAGS
 ac_ct_AR
 AR
+LINK_PYTHON_OBJS
+LINK_PYTHON_DEPS
+LIBRARY_DEPS
+STATIC_LIBPYTHON
 GNULD
 EXPORTSFROM
 EXPORTSYMS
@@ -1007,6 +1009,7 @@ with_cxx_main
 with_emscripten_target
 with_suffix
 enable_shared
+with_static_libpython
 enable_profiling
 with_pydebug
 with_trace_refs
@@ -1048,7 +1051,6 @@ with_openssl_rpath
 with_ssl_default_suites
 with_builtin_hashlib_hashes
 with_experimental_isolated_subinterpreters
-with_static_libpython
 enable_test_modules
 '
       ac_precious_vars='build_alias
@@ -1758,6 +1760,9 @@ Optional Packages:
                           Emscripten platform
   --with-suffix=SUFFIX    set executable suffix to SUFFIX (default is empty,
                           yes is mapped to '.exe')
+  --without-static-libpython
+                          do not build libpythonMAJOR.MINOR.a and do not
+                          install python.o (default is yes)
   --with-pydebug          build with Py_DEBUG defined (default is no)
   --with-trace-refs       enable tracing references for debugging purpose
                           (default is no)
@@ -1840,9 +1845,6 @@ Optional Packages:
   --with-experimental-isolated-subinterpreters
                           better isolate subinterpreters, experimental build
                           mode (default is no)
-  --without-static-libpython
-                          do not build libpythonMAJOR.MINOR.a and do not
-                          install python.o (default is yes)
 
 Some influential environment variables:
   PKG_CONFIG  path to pkg-config utility
@@ -6428,6 +6430,30 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_shared" >&5
 $as_echo "$enable_shared" >&6; }
 
+# --with-static-libpython
+STATIC_LIBPYTHON=1
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-static-libpython" >&5
+$as_echo_n "checking for --with-static-libpython... " >&6; }
+
+# Check whether --with-static-libpython was given.
+if test "${with_static_libpython+set}" = set; then :
+  withval=$with_static_libpython;
+if test "$withval" = no
+then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; };
+  STATIC_LIBPYTHON=0
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; };
+fi
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+fi
+
+
+
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-profiling" >&5
 $as_echo_n "checking for --enable-profiling... " >&6; }
 # Check whether --enable-profiling was given.
@@ -6550,6 +6576,31 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5
 $as_echo "$LDLIBRARY" >&6; }
 
+# LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable
+LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'
+LINK_PYTHON_DEPS='$(LIBRARY_DEPS)'
+if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
+    LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS"
+    if test "$STATIC_LIBPYTHON" = 1; then
+        LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
+    fi
+    # Link Python program to the shared library
+    LINK_PYTHON_OBJS='$(BLDLIBRARY)'
+else
+    if test "$STATIC_LIBPYTHON" = 0; then
+        # Build Python needs object files but don't need to build
+        # Python static library
+        LINK_PYTHON_DEPS="$LIBRARY_DEPS \$(LIBRARY_OBJS)"
+    fi
+    LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
+    # Link Python program to object files
+    LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
+fi
+
+
+
+
+# ar program
 
 if test -n "$ac_tool_prefix"; then
   for ac_prog in ar aal
@@ -21213,48 +21264,6 @@ $as_echo "no" >&6; }
 fi
 
 
-# --with-static-libpython
-STATIC_LIBPYTHON=1
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-static-libpython" >&5
-$as_echo_n "checking for --with-static-libpython... " >&6; }
-
-# Check whether --with-static-libpython was given.
-if test "${with_static_libpython+set}" = set; then :
-  withval=$with_static_libpython;
-if test "$withval" = no
-then
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; };
-  STATIC_LIBPYTHON=0
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; };
-fi
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-fi
-
-LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'
-if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
-    LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS"
-    if test "$STATIC_LIBPYTHON" = 1; then
-        LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
-    fi
-else
-    LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
-fi
-
-case $ac_sys_system/$ac_sys_emscripten_target in #(
-  Emscripten/browser) :
-    LIBRARY_DEPS="$LIBRARY_DEPS \$(WASM_STDLIB)" ;; #(
-  *) :
-     ;;
-esac
-
-
-
-
 # Check whether to disable test modules. Once set, setup.py will not build
 # test extension modules and "make install" will not install test suites.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --disable-test-modules" >&5

diff --git a/configure.ac b/configure.ac
index e5ebf7bc2e07aa4c029f64a54dcccaaf51893c53..89041b205f50d48360240659b40638a693cdb4f7 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -1231,6 +1231,23 @@ then
 fi
 AC_MSG_RESULT($enable_shared)
 
+# --with-static-libpython
+STATIC_LIBPYTHON=1
+AC_MSG_CHECKING(for --with-static-libpython)
+AC_ARG_WITH(static-libpython,
+  AS_HELP_STRING([--without-static-libpython],
+                 [do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes)]),
+[
+if test "$withval" = no
+then
+  AC_MSG_RESULT(no);
+  STATIC_LIBPYTHON=0
+else
+  AC_MSG_RESULT(yes);
+fi],
+[AC_MSG_RESULT(yes)])
+AC_SUBST(STATIC_LIBPYTHON)
+
 AC_MSG_CHECKING(for --enable-profiling)
 AC_ARG_ENABLE(profiling,
               AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
@@ -1336,6 +1353,31 @@ fi
 
 AC_MSG_RESULT($LDLIBRARY)
 
+# LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable
+LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'
+LINK_PYTHON_DEPS='$(LIBRARY_DEPS)'
+if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
+    LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS"
+    if test "$STATIC_LIBPYTHON" = 1; then
+        LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
+    fi
+    # Link Python program to the shared library
+    LINK_PYTHON_OBJS='$(BLDLIBRARY)'
+else
+    if test "$STATIC_LIBPYTHON" = 0; then
+        # Build Python needs object files but don't need to build
+        # Python static library
+        LINK_PYTHON_DEPS="$LIBRARY_DEPS \$(LIBRARY_OBJS)"
+    fi
+    LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
+    # Link Python program to object files
+    LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
+fi
+AC_SUBST(LIBRARY_DEPS)
+AC_SUBST(LINK_PYTHON_DEPS)
+AC_SUBST(LINK_PYTHON_OBJS)
+
+# ar program
 AC_SUBST(AR)
 AC_CHECK_TOOLS(AR, ar aal, ar)
 
@@ -6273,39 +6315,6 @@ else
 fi],
 [AC_MSG_RESULT(no)])
 
-# --with-static-libpython
-STATIC_LIBPYTHON=1
-AC_MSG_CHECKING(for --with-static-libpython)
-AC_ARG_WITH(static-libpython,
-  AS_HELP_STRING([--without-static-libpython],
-                 [do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes)]),
-[
-if test "$withval" = no
-then
-  AC_MSG_RESULT(no);
-  STATIC_LIBPYTHON=0
-else
-  AC_MSG_RESULT(yes);
-fi],
-[AC_MSG_RESULT(yes)])
-LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'
-if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
-    LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS"
-    if test "$STATIC_LIBPYTHON" = 1; then
-        LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
-    fi
-else
-    LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
-fi
-
-dnl browser needs a WASM assets stdlib bundle
-AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
-  [Emscripten/browser], [LIBRARY_DEPS="$LIBRARY_DEPS \$(WASM_STDLIB)"],
-)
-
-AC_SUBST(STATIC_LIBPYTHON)
-AC_SUBST(LIBRARY_DEPS)
-
 # Check whether to disable test modules. Once set, setup.py will not build
 # test extension modules and "make install" will not install test suites.
 AC_MSG_CHECKING(for --disable-test-modules)