AutoFDO: Enable machine function split optimization for AutoFDO

Enable the machine function split optimization for AutoFDO in Clang.

Machine function split (MFS) is a pass in the Clang compiler that
splits a function into hot and cold parts. The linker groups all
cold blocks across functions together. This decreases hot code
fragmentation and improves iCache and iTLB utilization.

MFS requires a profile so this is enabled only for the AutoFDO builds.

Co-developed-by: Han Shen <shenhan@google.com>
Signed-off-by: Han Shen <shenhan@google.com>
Signed-off-by: Rong Xu <xur@google.com>
Suggested-by: Sriraman Tallam <tmsriram@google.com>
Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com>
Tested-by: Yonghong Song <yonghong.song@linux.dev>
Tested-by: Yabin Cui <yabinc@google.com>
Tested-by: Nathan Chancellor <nathan@kernel.org>
Reviewed-by: Kees Cook <kees@kernel.org>
Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index bd64fdedabd2f67cb5d5b176e199aecddc2ed977..8a0bb3946cf053f9b737c01b62445d81a10728bd 100644 (file)
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -556,6 +556,11 @@ defined(CONFIG_AUTOFDO_CLANG)
                __cpuidle_text_end = .;                                 \
                __noinstr_text_end = .;
 
+#define TEXT_SPLIT                                                     \
+               __split_text_start = .;                                 \
+               *(.text.split .text.split.[0-9a-zA-Z_]*)                \
+               __split_text_end = .;
+
 #define TEXT_UNLIKELY                                                  \
                __unlikely_text_start = .;                              \
                *(.text.unlikely .text.unlikely.*)                      \
@@ -582,6 +587,7 @@ defined(CONFIG_AUTOFDO_CLANG)
                ALIGN_FUNCTION();                                       \
                *(.text.asan.* .text.tsan.*)                            \
                *(.text.unknown .text.unknown.*)                        \
+               TEXT_SPLIT                                              \
                TEXT_UNLIKELY                                           \
                . = ALIGN(PAGE_SIZE);                                   \
                TEXT_HOT                                                \
@@ -589,7 +595,6 @@ defined(CONFIG_AUTOFDO_CLANG)
                NOINSTR_TEXT                                            \
                *(.ref.text)
 
-
 /* sched.text is aling to function alignment to secure we have same
  * address even at second ld pass when generating System.map */
 #define SCHED_TEXT                                                     \

diff --git a/scripts/Makefile.autofdo b/scripts/Makefile.autofdo
index 6155d6fc4ca7fc50d3e23366e3f2021bcd6265a9..1caf2457e585c03123929fdade53cd1768c0fb82 100644 (file)
--- a/scripts/Makefile.autofdo
+++ b/scripts/Makefile.autofdo
@@ -10,6 +10,7 @@ endif
 
 ifdef CLANG_AUTOFDO_PROFILE
   CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections
+  CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions
 endif
 
 ifdef CONFIG_LTO_CLANG_THIN
@@ -17,6 +18,7 @@ ifdef CONFIG_LTO_CLANG_THIN
     KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE)
   endif
   KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto
+  KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions
 endif
 
 export CFLAGS_AUTOFDO_CLANG