From 60c3d0446546332e1645541f2dc2c072cd019fe9 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Fri, 6 Sep 2024 10:26:06 +0200 Subject: [PATCH] autotools: add support for 'unity' builds, enable in CI Implement the "unity" builds as known from CMake, but for autotools. It's limited to `lib` and `src` (CMake also supports it in `tests`). Enable with: `--enable-unity` (disabled by default) Unity builds speed up builds significantly. Cygwin and Windows builds in particular, but the effect is noticeable on most systems. It also allows discovering unity issues with autotools, benefitting also CMake when building the same combination. In CI it makes turnaround times quicker. This closes build performance with CMake. autotools still lags behind because it builds shared and static libcurl in two, separate passes. CMake does it in one. Manpage compilation isn't batched, it is in CMake. After unity and test bundle support the slowest parts of the build are the configuration phase (which is effectively a tedious, non-parallel, compilation and/or linking of 300+ tiny programs. The next bottleneck is compiling individual examples and finally test servers (only slow with autotools). The autotools implementation is slightly less efficient than CMake, because 3 sources are permanently excluded while in CMake this isn't necessary and solved more efficiently while building libtests. There is also no 'unity' support for tests, making them a less efficient also. Enable it in CI for most `configure` jobs. Except in GHA/dist (though it works fine there too), to use the default config there. Also skip for the Linux AWC-LC job where it made builds time a few seconds longer (reason undiscovered.) Autotools test suite builds compared between master -> `--enable-unity`: - GHA/Linux: 32s -> 12s https://github.com/curl/curl/actions/runs/10705668823/job/29681617374 https://github.com/curl/curl/actions/runs/10742978889/job/29796766297 - GHA/macOS: 37s -> 10s https://github.com/curl/curl/actions/runs/10705668813/job/29681632885 https://github.com/curl/curl/actions/runs/10742978699/job/29796768875 - GHA/FreeBSD: 15m25 -> 10m58 (full workflow time, ~qemu) https://github.com/curl/curl/actions/runs/10705668811/job/29681607915 https://github.com/curl/curl/actions/runs/10742978937/job/29796766115 - GHA/Cygwin: 3m32 -> 1m21 https://github.com/curl/curl/actions/runs/10705668809/job/29681609965 https://github.com/curl/curl/actions/runs/10742978645/job/29796756933 - GHA/MSYS2: 2m42 -> 50s https://github.com/curl/curl/actions/runs/10705668808/job/29681621166 https://github.com/curl/curl/actions/runs/10742978662/job/29799739289 - GHA/mingw-w64: 5m32 -> 1m23 https://github.com/curl/curl/actions/runs/10705668808/job/29681628787 https://github.com/curl/curl/actions/runs/10742978662/job/29799741568 Closes #14815 --- .circleci/config.yml | 12 +++--- .github/workflows/cygwin.yml | 2 +- .github/workflows/http3-linux.yml | 2 +- .github/workflows/linux-old.yml | 4 +- .github/workflows/linux.yml | 8 ++-- .github/workflows/linux32.yml | 4 +- .github/workflows/macos.yml | 4 +- .github/workflows/non-native.yml | 4 +- .github/workflows/torture.yml | 4 +- .github/workflows/windows.yml | 4 +- .github/workflows/wolfssl.yml | 2 +- configure.ac | 23 +++++++++++ lib/Makefile.am | 19 ++++++++++ scripts/Makefile.am | 2 +- scripts/mk-unity.pl | 63 +++++++++++++++++++++++++++++++ src/Makefile.am | 33 +++++++++++++++- 16 files changed, 167 insertions(+), 23 deletions(-) create mode 100755 scripts/mk-unity.pl diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c705411e6..a0e802909d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ commands: - run: command: | autoreconf -fi - ./configure --disable-dependency-tracking --enable-warnings --enable-werror --with-openssl \ + ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror --with-openssl \ || { tail -1000 config.log; false; } configure-openssl-no-verbose: @@ -41,7 +41,7 @@ commands: - run: command: | autoreconf -fi - ./configure --disable-dependency-tracking --disable-verbose --enable-werror --with-openssl \ + ./configure --disable-dependency-tracking --enable-unity --disable-verbose --enable-werror --with-openssl \ || { tail -1000 config.log; false; } configure-no-proxy: @@ -49,7 +49,7 @@ commands: - run: command: | autoreconf -fi - ./configure --disable-dependency-tracking --disable-proxy --enable-werror --with-openssl \ + ./configure --disable-dependency-tracking --enable-unity --disable-proxy --enable-werror --with-openssl \ || { tail -1000 config.log; false; } install-cares: @@ -110,7 +110,7 @@ commands: - run: command: | autoreconf -fi - ./configure --disable-dependency-tracking --enable-warnings --enable-werror --with-openssl --enable-ares \ + ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror --with-openssl --enable-ares \ || { tail -1000 config.log; false; } configure-wolfssh: @@ -118,7 +118,7 @@ commands: - run: command: | autoreconf -fi - LDFLAGS="-Wl,-rpath,$HOME/wssh/lib" ./configure --disable-dependency-tracking --enable-warnings --enable-werror --with-wolfssl=$HOME/wssl --with-wolfssh=$HOME/wssh \ + LDFLAGS="-Wl,-rpath,$HOME/wssh/lib" ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror --with-wolfssl=$HOME/wssl --with-wolfssh=$HOME/wssh \ || { tail -1000 config.log; false; } configure-cares-debug: @@ -126,7 +126,7 @@ commands: - run: command: | autoreconf -fi - ./configure --disable-dependency-tracking --enable-debug --enable-werror --with-openssl --enable-ares \ + ./configure --disable-dependency-tracking --enable-unity --enable-debug --enable-werror --with-openssl --enable-ares \ || { tail -1000 config.log; false; } build: diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml index 1c2fd19ec2..7159314155 100644 --- a/.github/workflows/cygwin.yml +++ b/.github/workflows/cygwin.yml @@ -81,7 +81,7 @@ jobs: timeout-minutes: 5 run: | PATH="/usr/bin:$(cygpath "${SYSTEMROOT}")/System32" - mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \ --prefix="${HOME}"/install \ --enable-websockets \ --with-openssl \ diff --git a/.github/workflows/http3-linux.yml b/.github/workflows/http3-linux.yml index 7132e41306..d58f64ea1e 100644 --- a/.github/workflows/http3-linux.yml +++ b/.github/workflows/http3-linux.yml @@ -451,7 +451,7 @@ jobs: - run: autoreconf -fi name: 'autoreconf' - - run: ./configure --disable-dependency-tracking ${{ matrix.build.configure }} + - run: ./configure --disable-dependency-tracking --enable-unity ${{ matrix.build.configure }} name: 'configure' - run: make V=1 diff --git a/.github/workflows/linux-old.yml b/.github/workflows/linux-old.yml index 17c99a4426..5108e80961 100644 --- a/.github/workflows/linux-old.yml +++ b/.github/workflows/linux-old.yml @@ -113,7 +113,9 @@ jobs: run: | mkdir bld-am cd bld-am - ../configure --disable-dependency-tracking --enable-warnings --enable-werror --with-openssl --enable-ares --with-libssh --with-zstd --with-gssapi --prefix="$PWD"/../install-am + ../configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \ + --with-openssl --enable-ares --with-libssh --with-zstd --with-gssapi \ + --prefix="$PWD"/../install-am - name: 'autoconf build' run: | diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 6b099c4161..dcca9e515e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -496,7 +496,10 @@ jobs: if: ${{ matrix.build.configure }} name: 'autoreconf' - - run: ${{ matrix.build.configure-prefix }} ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }} + - run: | + ${{ matrix.build.configure-prefix }} \ + ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \ + ${{ matrix.build.configure }} if: ${{ matrix.build.configure }} name: 'configure (autotools)' @@ -555,8 +558,7 @@ jobs: - if: contains(matrix.build.install_steps, 'pytest') # run for `tests/http` directory, so pytest does not pick up any other # packages we might have built here - run: - pytest -v tests/http + run: pytest -v tests/http name: 'run pytest' env: TFLAGS: "${{ matrix.build.tflags }}" diff --git a/.github/workflows/linux32.yml b/.github/workflows/linux32.yml index da146c2b6a..43494f289c 100644 --- a/.github/workflows/linux32.yml +++ b/.github/workflows/linux32.yml @@ -69,7 +69,9 @@ jobs: - run: autoreconf -fi name: 'autoreconf' - - run: ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }} + - run: | + ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \ + ${{ matrix.build.configure }} name: 'configure' - run: make V=1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 84e0fb1618..21f61dc95d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -203,7 +203,7 @@ jobs: options+=" --with-sysroot=$(xcrun --sdk macosx --show-sdk-path 2>/dev/null)" CFLAGS+=" --sysroot=$(xcrun --sdk macosx --show-sdk-path 2>/dev/null)" fi - mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \ --disable-dependency-tracking \ --with-libpsl=$(brew --prefix libpsl) \ ${{ matrix.configure }} ${options} @@ -605,7 +605,7 @@ jobs: [ '${{ matrix.config }}' = 'SecureTransport' ] && options+=' --with-secure-transport' CFLAGS+=' -mmacosx-version-min=${{ matrix.macos-version-min }}' # would pick up nghttp2, libidn2, but libssh2 is disabled by default - mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \ --disable-dependency-tracking \ --disable-docs --disable-manual \ --without-nghttp2 --without-libidn2 \ diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml index 0a98ebf0d3..cfa030a33a 100644 --- a/.github/workflows/non-native.yml +++ b/.github/workflows/non-native.yml @@ -136,7 +136,7 @@ jobs: pkgconf brotli openldap26-client libidn2 libnghttp2 nghttp2 stunnel py311-impacket autoreconf -fi export CC='${{ matrix.compiler }}' - mkdir bld && cd bld && ../configure --enable-debug --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-debug --enable-warnings --enable-werror \ --prefix="${HOME}"/install \ --enable-websockets \ --with-openssl \ @@ -198,7 +198,7 @@ jobs: run: | ln -s /usr/bin/gcpp /usr/bin/cpp # Some tests expect `cpp`, which is named `gcpp` in this env. autoreconf -fi - mkdir bld && cd bld && ../configure --enable-debug --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-debug --enable-warnings --enable-werror \ --prefix="${HOME}"/install \ --enable-websockets \ --with-openssl \ diff --git a/.github/workflows/torture.yml b/.github/workflows/torture.yml index 3558d27b03..88e71cfc5d 100644 --- a/.github/workflows/torture.yml +++ b/.github/workflows/torture.yml @@ -74,7 +74,9 @@ jobs: - run: autoreconf -fi name: 'autoreconf' - - run: ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }} + - run: | + ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \ + ${{ matrix.build.configure }} name: 'configure' - run: make V=1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 052a405602..e6f5e2c70e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -103,7 +103,7 @@ jobs: if: ${{ matrix.build == 'autotools' }} timeout-minutes: 5 run: | - mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \ --prefix="${HOME}"/install \ --enable-websockets \ --with-openssl \ @@ -415,7 +415,7 @@ jobs: - name: 'autotools configure' if: ${{ matrix.build == 'autotools' }} run: | - mkdir bld && cd bld && ../configure --enable-warnings --enable-werror \ + mkdir bld && cd bld && ../configure --enable-unity --enable-warnings --enable-werror \ --host=${TRIPLET} \ --with-schannel --with-winidn \ --without-libpsl \ diff --git a/.github/workflows/wolfssl.yml b/.github/workflows/wolfssl.yml index ba6d92859a..3f7a9ba4df 100644 --- a/.github/workflows/wolfssl.yml +++ b/.github/workflows/wolfssl.yml @@ -84,7 +84,7 @@ jobs: - run: autoreconf -fi name: 'autoreconf' - - run: ./configure --disable-dependency-tracking --enable-warnings --enable-werror ${{ matrix.build.configure }} + - run: ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror ${{ matrix.build.configure }} name: 'configure' - run: make V=1 diff --git a/configure.ac b/configure.ac index cec382a3d0..d6605fbb7c 100644 --- a/configure.ac +++ b/configure.ac @@ -605,6 +605,29 @@ if test "$curl_cv_native_windows" = "yes"; then [AC_MSG_ERROR([windres not found in PATH. Windows builds require windres. Cannot continue.])]) fi +dnl ---------------------------------------- +dnl whether use "unity" mode for lib and src +dnl ---------------------------------------- + +want_unity='no' +AC_MSG_CHECKING([whether to build libcurl and curl in "unity" mode]) +AC_ARG_ENABLE(unity, +AS_HELP_STRING([--enable-unity],[Enable unity mode]) +AS_HELP_STRING([--disable-unity],[Disable unity (default)]), +[ case "$enableval" in + yes) + want_unity='yes' + AC_MSG_RESULT([yes]) + ;; + *) + AC_MSG_RESULT([no]) + ;; + esac ], + AC_MSG_RESULT([no]) +) + +AM_CONDITIONAL([USE_UNITY], [test "$want_unity" = 'yes']) + dnl ************************************************************ dnl switch off particular protocols dnl diff --git a/lib/Makefile.am b/lib/Makefile.am index 851cee293a..3a40280b94 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -70,8 +70,27 @@ AM_CFLAGS = # Makefile.inc provides the CSOURCES and HHEADERS defines include Makefile.inc +if USE_UNITY +# Keep these separate to avoid duplicate definitions when linking libtests +# in static mode. +curl_EXCLUDE = curl_threads.c timediff.c warnless.c +if DEBUGBUILD +# We must compile these sources separately to avoid memdebug.h redefinitions +# applying to them. +curl_EXCLUDE += memdebug.c curl_multibyte.c +endif +libcurl_unity.c: $(top_srcdir)/scripts/mk-unity.pl $(CSOURCES) + @PERL@ $(top_srcdir)/scripts/mk-unity.pl $(srcdir) $(CSOURCES) --exclude $(curl_EXCLUDE) > libcurl_unity.c + +nodist_libcurl_la_SOURCES = libcurl_unity.c +libcurl_la_SOURCES = $(curl_EXCLUDE) +nodist_libcurlu_la_SOURCES = libcurl_unity.c +libcurlu_la_SOURCES = $(curl_EXCLUDE) +CLEANFILES = libcurl_unity.c +else libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS) libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS) +endif libcurl_la_CPPFLAGS_EXTRA = libcurl_la_LDFLAGS_EXTRA = diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 546041074e..02e1c91684 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -23,7 +23,7 @@ ########################################################################### EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl \ - mk-ca-bundle.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \ + mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \ dmaketgz maketgz release-tools.sh verify-release cmakelint.sh ZSH_FUNCTIONS_DIR = @ZSH_FUNCTIONS_DIR@ diff --git a/scripts/mk-unity.pl b/scripts/mk-unity.pl new file mode 100755 index 0000000000..f9ebf2524c --- /dev/null +++ b/scripts/mk-unity.pl @@ -0,0 +1,63 @@ +#!/usr/bin/env perl +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Viktor Szakats +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### + +# Helper script for "unity"-like support in autotools, to generate the umbrella +# C source that includes the individual source files. Reads Makefile.inc and +# accepts the variable name containing all the source files to include. Also +# allow a list of exceptions that are to be excluded from the generated file. + +use strict; +use warnings; + +if(!@ARGV) { + die "Usage: $0 [] [--exclude ]\n"; +} + +# Specific sources to exclude or add as an extra source file +my @src; +my %exclude; +my $in_exclude = 0; +foreach my $src (@ARGV) { + if($in_exclude) { + $exclude{$src} = 1; + } + elsif($src eq "--exclude") { + $in_exclude = 1; + } + else { + push @src, $src; + } +} + +print <
curltool_unity.c + +nodist_curl_SOURCES = curltool_unity.c +curl_SOURCES = $(curl_EXCLUDE) +CLEANFILES += curltool_unity.c +else # CURL_FILES comes from Makefile.inc curl_SOURCES = $(CURL_FILES) +endif if HAVE_WINDRES curl_SOURCES += $(CURL_RCFILES) $(CURL_RCFILES): tool_version.h @@ -84,8 +106,17 @@ libcurltool_la_CPPFLAGS = $(AM_CPPFLAGS) \ -DCURL_STATICLIB -DUNITTESTS libcurltool_la_CFLAGS = libcurltool_la_LDFLAGS = -static $(LINKFLAGS) +if USE_UNITY +libcurltool_unity.c: $(top_srcdir)/scripts/mk-unity.pl $(CURL_CFILES) $(CURLTOOL_LIBCURL_CFILES) + @PERL@ $(top_srcdir)/scripts/mk-unity.pl $(srcdir) $(CURL_CFILES) $(CURLTOOL_LIBCURL_CFILES) --exclude $(curl_EXCLUDE) > libcurltool_unity.c + +nodist_libcurltool_la_SOURCES = libcurltool_unity.c +libcurltool_la_SOURCES = $(curl_EXCLUDE) +CLEANFILES += libcurltool_unity.c +else libcurltool_la_SOURCES = $(CURL_FILES) endif +endif # Use absolute directory to disable VPATH ASCIIPAGE=$(top_builddir)/docs/cmdline-opts/curl.txt @@ -127,7 +158,7 @@ $(HUGE): echo '#include "tool_hugehelp.h"' >> $(HUGE) endif -CLEANFILES = $(HUGE) +CLEANFILES += $(HUGE) CA_EMBED_CSOURCE = tool_ca_embed.c CURL_CFILES += $(CA_EMBED_CSOURCE) -- 2.47.3