Build System: Improve ccache matching for different menuselect options.

Changing any Menuselect option in the `Compiler Flags` section causes a
full rebuild of the Asterisk source tree.  Every enabled option causes
a #define to be added to buildopts.h, thus breaking ccache caching for
every source file that includes "asterisk.h".  In most cases each option
only applies to one or two files.  Now we only define those options for
the specific sources which use them, this causes much better cache
matching when working with multiple builds.  For example testing code
with an without MALLOC_DEBUG will now use just over half the ccache
size, only main/astmm.o will have two builds cached instead of every
file.

Reorder main/Makefile so _ASTCFLAGS set on specific object files are all
together, sorted by filename.  Stop adding -DMALLOC_DEBUG to CFLAGS of
bundled pjproject, this define is no longer used by any header so only
serves to break cache.

The only code change is a slight adjustment to how main/astmm.c is
initialized.  Initialization functions always exist so main/asterisk.c
can call them unconditionally.  Additionally rename the astmm
initialization functions so they are not exported.

Change-Id: Ie2085237a964f6e1e6fff55ed046e2afff83c027

diff --git a/Makefile.rules b/Makefile.rules
index 7f917b479f076f5b41322b0cfcc7b16c2e95b3b1..b9799272b83223bd53b783f2b7027535cefa44fc 100644 (file)
--- a/Makefile.rules
+++ b/Makefile.rules
@@ -20,6 +20,9 @@
 # Helpful functions
 # call with $(call function,...)
 tolower = $(shell echo $(1) | tr '[:upper:]' '[:lower:]')
+# Takes a list of MENUSELECT_CFLAG Id and returns CFLAGS to declare
+# the ones which are enabled.
+get_menuselect_cflags=$(patsubst %,-D%,$(filter $1,$(MENUSELECT_CFLAGS)))
 
 .PHONY: dist-clean
 

diff --git a/build_tools/make_buildopts_h b/build_tools/make_buildopts_h
index 57df27dd46ec960480febaaccad7cf102608dda9..692e15b027b0703cfeba7e79ff590074be672962 100755 (executable)
--- a/build_tools/make_buildopts_h
+++ b/build_tools/make_buildopts_h
@@ -20,21 +20,30 @@ fi
 
 TMP=`${GREP} -e "^MENUSELECT_CFLAGS" menuselect.makeopts | sed 's/MENUSELECT_CFLAGS\=//g' | sed 's/-D//g'`
 for x in ${TMP}; do
-       echo "#define ${x} 1"
        if test "${x}" = "AO2_DEBUG" \
                        -o "${x}" = "BETTER_BACKTRACES" \
                        -o "${x}" = "BUILD_NATIVE" \
+                       -o "${x}" = "COMPILE_DOUBLE" \
+                       -o "${x}" = "DEBUG_CHAOS" \
                        -o "${x}" = "DEBUG_SCHEDULER" \
                        -o "${x}" = "DETECT_DEADLOCKS" \
                        -o "${x}" = "DONT_OPTIMIZE" \
                        -o "${x}" = "DUMP_SCHEDULER" \
                        -o "${x}" = "LOTS_OF_SPANS" \
-                       -o "${x}" = "LOW_MEMORY" \
                        -o "${x}" = "MALLOC_DEBUG" \
                        -o "${x}" = "RADIO_RELAX" \
                        -o "${x}" = "REBUILD_PARSERS" \
-                       -o "${x}" = "REF_DEBUG" ; then
-               # These aren't ABI affecting options, keep them out of AST_BUILDOPTS
+                       -o "${x}" = "REF_DEBUG" \
+                       -o "${x}" = "USE_HOARD_ALLOCATOR" ; then
+               # These options are only for specific sources and have no effect on public ABI.
+               # Keep them out of buildopts.h so ccache does not invalidate all sources.
+               continue
+       fi
+
+       echo "#define ${x} 1"
+       if test "${x}" = "LOW_MEMORY" ; then
+               # LOW_MEMORY isn't an ABI affecting option but it is used in many sources
+               # so it gets defined globally but is not included in AST_BUILTOPTS.
                continue
        fi
        if test "x${BUILDOPTS}" != "x" ; then

diff --git a/channels/Makefile b/channels/Makefile
index 6398d9599a3fa886492dd86784d3be3e4385988e..96421d82b03a5c3ee9011fc0821d9d718b878c2e 100644 (file)
--- a/channels/Makefile
+++ b/channels/Makefile
@@ -29,6 +29,7 @@ $(call MOD_ADD_C,chan_pjsip,$(wildcard pjsip/*.c))
 $(call MOD_ADD_C,chan_dahdi,$(wildcard dahdi/*.c) sig_analog.c sig_pri.c sig_ss7.c)
 $(call MOD_ADD_C,chan_misdn,misdn_config.c misdn/isdn_lib.c misdn/isdn_msg_parser.c)
 
+chan_dahdi.o: _ASTCFLAGS+=$(call get_menuselect_cflags,LOTS_OF_SPANS)
 chan_mgcp.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION)
 chan_unistim.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION)
 chan_phone.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION)

diff --git a/include/asterisk/_private.h b/include/asterisk/_private.h
index 89a8f54b6c9c202746b68dc90c86db0f65957912..d9547681aaed244ba1817f07efe4d012545f404c 100644 (file)
--- a/include/asterisk/_private.h
+++ b/include/asterisk/_private.h
@@ -57,6 +57,22 @@ void ast_msg_shutdown(void);        /*!< Provided by message.c */
 int aco_init(void);             /*!< Provided by config_options.c */
 int dns_core_init(void);        /*!< Provided by dns_core.c */
 
+/*!
+ * \brief Initialize malloc debug phase 1.
+ *
+ * \note Must be called first thing after forking.
+ *
+ * \return Nothing
+ */
+void load_astmm_phase_1(void);
+
+/*!
+ * \brief Initialize malloc debug phase 2.
+ *
+ * \return Nothing
+ */
+void load_astmm_phase_2(void);
+
 /*!
  * \brief Initialize the bridging system.
  * \since 12.0.0

diff --git a/include/asterisk/astmm.h b/include/asterisk/astmm.h
index 13ebe030c10de9db6251be9d4b31c9e9477218b5..e1f91ddeab76c2521177b154f0ab6bbc0eb839c0 100644 (file)
--- a/include/asterisk/astmm.h
+++ b/include/asterisk/astmm.h
@@ -31,13 +31,6 @@ extern "C" {
 #define _ASTERISK_ASTMM_H
 /* IWYU pragma: private, include "asterisk.h" */
 
-#if defined(MALLOC_DEBUG) && !defined(STANDALONE) && !defined(STANDALONE2)
-#define __AST_DEBUG_MALLOC
-
-void __ast_mm_init_phase_1(void);
-void __ast_mm_init_phase_2(void);
-#endif
-
 void *ast_std_malloc(size_t size) attribute_malloc;
 void *ast_std_calloc(size_t nmemb, size_t size) attribute_malloc;
 void *ast_std_realloc(void *ptr, size_t size);

diff --git a/main/Makefile b/main/Makefile
index 0e29c843f060f85ace9dd8f68088396ae5e06cee..1cb2c25eff5aa9ffbce1cec53b8f105a48b2526e 100644 (file)
--- a/main/Makefile
+++ b/main/Makefile
@@ -141,21 +141,33 @@ endif
        $(CMD_PREFIX) cat $@.fix >> $@
        $(CMD_PREFIX) rm $@.fix
 
-ast_expr2f.o: _ASTCFLAGS+=-Wno-unused
-cdr.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION)
+ifneq ($(findstring ENABLE_UPLOADS,$(MENUSELECT_CFLAGS)),)
+GMIMELDFLAGS+=$(GMIME_LIB)
+GMIMECFLAGS+=$(GMIME_INCLUDE)
+endif
+
+# Alter CFLAGS for specific sources
+stdtime/localtime.o: _ASTCFLAGS+=$(AST_NO_STRICT_OVERFLOW) -Wno-format-nonliteral
 
-db.o: _ASTCFLAGS+=$(SQLITE3_INCLUDE)
 asterisk.o: _ASTCFLAGS+=$(LIBEDIT_INCLUDE)
-json.o: _ASTCFLAGS+=$(JANSSON_INCLUDE)
+ast_expr2f.o: _ASTCFLAGS+=-Wno-unused
+astmm.o: _ASTCFLAGS+=$(call get_menuselect_cflags,MALLOC_DEBUG DEBUG_CHAOS)
+astobj2.o astobj2_container.o astobj2_hash.o astobj2_rbtree.o: _ASTCFLAGS+=$(call get_menuselect_cflags,AO2_DEBUG)
+backtrace.o: _ASTCFLAGS+=$(call get_menuselect_cflags,BETTER_BACKTRACES)
 bucket.o: _ASTCFLAGS+=$(URIPARSER_INCLUDE)
+cdr.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION)
 crypt.o: _ASTCFLAGS+=$(CRYPT_INCLUDE)
+db.o: _ASTCFLAGS+=$(SQLITE3_INCLUDE)
+dsp.o: _ASTCFLAGS+=$(call get_menuselect_cflags,RADIO_RELAX)
+http.o: _ASTCFLAGS+=$(GMIMECFLAGS)
+iostream.o: _ASTCFLAGS+=$(OPENSSL_INCLUDE)
+json.o: _ASTCFLAGS+=$(JANSSON_INCLUDE)
+lock.o: _ASTCFLAGS+=$(call get_menuselect_cflags,DETECT_DEADLOCKS)
+options.o: _ASTCFLAGS+=$(call get_menuselect_cflags,REF_DEBUG)
+sched.o: _ASTCFLAGS+=$(call get_menuselect_cflags,DEBUG_SCHEDULER DUMP_SCHEDULER)
+tcptls.o: _ASTCFLAGS+=$(OPENSSL_INCLUDE) -Wno-deprecated-declarations
 uuid.o: _ASTCFLAGS+=$(UUID_INCLUDE)
 
-ifneq ($(findstring ENABLE_UPLOADS,$(MENUSELECT_CFLAGS)),)
-http.o: _ASTCFLAGS+=$(GMIME_INCLUDE)
-endif
-
-stdtime/localtime.o: _ASTCFLAGS+=$(AST_NO_STRICT_OVERFLOW) -Wno-format-nonliteral
 
 OBJS:=$(sort $(OBJS))
 
@@ -169,10 +181,6 @@ else
 MAIN_TGT:=asterisk
 endif
 
-ifneq ($(findstring ENABLE_UPLOADS,$(MENUSELECT_CFLAGS)),)
-GMIMELDFLAGS+=$(GMIME_LIB)
-endif
-
 $(OBJS): _ASTCFLAGS+=-DAST_MODULE=\"core\" -DAST_IN_CORE
 $(MOD_OBJS): _ASTCFLAGS+=$(call MOD_ASTCFLAGS,$*)
 
@@ -306,9 +314,6 @@ endif
 
 endif
 
-iostream.o: _ASTCFLAGS+=$(OPENSSL_INCLUDE)
-tcptls.o: _ASTCFLAGS+=$(OPENSSL_INCLUDE) -Wno-deprecated-declarations
-
 $(MAIN_TGT): $(OBJS) $(MOD_OBJS) $(ASTSSL_LIB) $(ASTPJ_LIB)
        @$(CC) -c -o buildinfo.o $(_ASTCFLAGS) buildinfo.c $(ASTCFLAGS)
        $(ECHO_PREFIX) echo "   [LD] $(OBJS) $(MOD_OBJS) -> $@"

diff --git a/main/asterisk.c b/main/asterisk.c
index 8fa1c19a42d27784a99726c4411d8f58f3d65f7f..335472490c0c3b3d2f6c207029129c4dec8c4d22 100644 (file)
--- a/main/asterisk.c
+++ b/main/asterisk.c
@@ -3961,9 +3961,7 @@ static void asterisk_daemon(int isroot, const char *runuser, const char *rungrou
         * an Asterisk instance, and that there isn't one already running. */
        multi_thread_safe = 1;
 
-#if defined(__AST_DEBUG_MALLOC)
-       __ast_mm_init_phase_1();
-#endif /* defined(__AST_DEBUG_MALLOC) */
+       load_astmm_phase_1();
 
        /* Check whether high prio was succesfully set by us or some
         * other incantation. */
@@ -4168,9 +4166,7 @@ static void asterisk_daemon(int isroot, const char *runuser, const char *rungrou
 
        pthread_sigmask(SIG_UNBLOCK, &sigs, NULL);
 
-#if defined(__AST_DEBUG_MALLOC)
-       __ast_mm_init_phase_2();
-#endif /* defined(__AST_DEBUG_MALLOC) */
+       load_astmm_phase_2();
 
        ast_cli_register_multiple(cli_asterisk_shutdown, ARRAY_LEN(cli_asterisk_shutdown));
        ast_cli_register_multiple(cli_asterisk, ARRAY_LEN(cli_asterisk));

diff --git a/main/astmm.c b/main/astmm.c
index 08f67ab3365d7aca1afdd33533e750d96ddb0132..c845ee7c7a5bf111a234f6f60fce83e1ac1b7ce3 100644 (file)
--- a/main/astmm.c
+++ b/main/astmm.c
@@ -31,6 +31,7 @@
 #define ASTMM_LIBC ASTMM_IGNORE
 #include "asterisk.h"
 
+#include "asterisk/_private.h"
 #include "asterisk/logger.h"
 
 /*!
@@ -62,6 +63,10 @@
 #define ast_log_safe ast_log
 #endif
 
+#if defined(MALLOC_DEBUG) && !defined(STANDALONE) && !defined(STANDALONE2)
+#define __AST_DEBUG_MALLOC
+#endif
+
 #define MALLOC_FAILURE_MSG \
        ast_log_safe(LOG_ERROR, "Memory Allocation Failure in function %s at line %d of %s\n", func, lineno, file)
 
@@ -1501,14 +1506,7 @@ static void mm_atexit_final(void)
        }
 }
 
-/*!
- * \brief Initialize malloc debug phase 1.
- *
- * \note Must be called first thing in main().
- *
- * \return Nothing
- */
-void __ast_mm_init_phase_1(void)
+void load_astmm_phase_1(void)
 {
        atexit(mm_atexit_final);
 }
@@ -1522,12 +1520,7 @@ static void mm_atexit_ast(void)
        ast_cli_unregister_multiple(cli_memory, ARRAY_LEN(cli_memory));
 }
 
-/*!
- * \brief Initialize malloc debug phase 2.
- *
- * \return Nothing
- */
-void __ast_mm_init_phase_2(void)
+void load_astmm_phase_2(void)
 {
        char filename[PATH_MAX];
 
@@ -1550,6 +1543,14 @@ void __ast_mm_init_phase_2(void)
 
 #else  /* !defined(__AST_DEBUG_MALLOC) */
 
+void load_astmm_phase_1(void)
+{
+}
+
+void load_astmm_phase_2(void)
+{
+}
+
 void *__ast_repl_calloc(size_t nmemb, size_t size, const char *file, int lineno, const char *func)
 {
        DEBUG_CHAOS_RETURN(DEBUG_CHAOS_ALLOC_CHANCE, NULL);

diff --git a/tests/Makefile b/tests/Makefile
index f64669bcdc898ee438dfe9cf4dbafeeea35df466..715c3f859387ff8dd4a9b777ba7ada0c0646c77a 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -19,5 +19,6 @@ all: _all
 
 include $(ASTTOPDIR)/Makefile.moddir_rules
 
+test_astobj2.o: _ASTCFLAGS+=$(call get_menuselect_cflags,AO2_DEBUG)
 test_strings.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION) $(AST_NO_STRINGOP_TRUNCATION)
 test_voicemail_api.o: _ASTCFLAGS+=$(AST_NO_FORMAT_TRUNCATION)

diff --git a/third-party/pjproject/Makefile b/third-party/pjproject/Makefile
index 476eb44017953b3da0481f5c7305f86a9f360913..c761cb59c7c1d6dad62514340ae5026ea8129502 100644 (file)
--- a/third-party/pjproject/Makefile
+++ b/third-party/pjproject/Makefile
@@ -64,9 +64,6 @@ ifeq ($(SPECIAL_TARGETS),)
         ifeq ($(AST_DEVMODE),yes)
             CF += -DPJPROJECT_BUNDLED_ASSERTIONS=yes
         endif
-        ifeq ($(findstring MALLOC_DEBUG,$(MENUSELECT_CFLAGS)),MALLOC_DEBUG)
-            CF += -DMALLOC_DEBUG
-        endif
         MALLOC_DEBUG_LIBS = source/pjsip-apps/lib/libasterisk_malloc_debug.a
         MALLOC_DEBUG_LDFLAGS = -L$(PJDIR)/pjsip-apps/lib -Wl,-whole-archive -lasterisk_malloc_debug -Wl,-no-whole-archive
         ifeq ($(findstring DONT_OPTIMIZE,$(MENUSELECT_CFLAGS)),)

diff --git a/utils/Makefile b/utils/Makefile
index 2ec1b917c6f8c86695591a4d4e5097b8e8881677..738d1e42cd57a8424de9d5e12d901592b9e8691f 100644 (file)
--- a/utils/Makefile
+++ b/utils/Makefile
@@ -172,6 +172,7 @@ threadstorage.c: $(ASTTOPDIR)/main/threadstorage.c
        $(CMD_PREFIX) cp "$<" "$@"
 
 
+extconf.o: _ASTCFLAGS+=$(call get_menuselect_cflags,DETECT_DEADLOCKS)
 extconf.o: extconf.c
 
 conf2ael: LIBS+=$(AST_CLANG_BLOCKS_LIBS)