From: Gary Guo Date: Tue, 3 Feb 2026 11:34:10 +0000 (+0000) Subject: kbuild: rust: provide an option to inline C helpers into Rust X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=3a2486cc1da5cf637fe5da4540929d67c4540022;p=thirdparty%2Flinux.git kbuild: rust: provide an option to inline C helpers into Rust A new experimental Kconfig option, `RUST_INLINE_HELPERS` is added to allow C helpers (which were created to allow Rust to call into inline/macro C functions without having to re-implement the logic in Rust) to be inlined into Rust crates without performing global LTO. If the option is enabled, the following is performed: * For helpers, instead of compiling them to an object file to be linked into vmlinux, they're compiled to LLVM IR bitcode. Two versions are generated: one for built-in code (`helpers.bc`) and one for modules (`helpers_module.bc`, with -DMODULE defined). This ensures that C macros/inlines that behave differently for modules (e.g. static calls) function correctly when inlined. * When a Rust crate or object is compiled, instead of generating an object file, LLVM bitcode is generated. * llvm-link is invoked with --internalize to combine the helper bitcode with the crate bitcode. This step is similar to LTO, but this is much faster since it only needs to inline the helpers. * clang is invoked to turn the combined bitcode into a final object file. * Since clang may produce LLVM bitcode when LTO is enabled, and objtool requires ELF input, $(cmd_ld_single) is invoked to ensure the object is converted to ELF before objtool runs. The --internalize flag tells llvm-link to treat all symbols in helpers.bc using `internal` linkage [1]. This matches the behavior of `clang` on `static inline` functions, and avoids exporting the symbol from the object file. To ensure that RUST_INLINE_HELPERS is not incompatible with BTF, we pass the -g0 flag when building helpers. See commit 5daa0c35a1f0 ("rust: Disallow BTF generation with Rust + LTO") for details. We have an intended triple mismatch of `aarch64-unknown-none` vs `aarch64-unknown-linux-gnu`, so we pass --suppress-warnings to llvm-link to suppress it. I considered adding some sort of check that KBUILD_MODNAME is not present in helpers_module.bc, but this is actually not so easy to carry out because .bc files store strings in a weird binary format, so you cannot just grep it for a string to check whether it ended up using KBUILD_MODNAME anywhere. [ Andreas writes: For the rnull driver, enabling helper inlining with this patch gives an average speedup of 2% over the set of 120 workloads that we publish on [2]. Link: https://rust-for-linux.com/null-block-driver [2] This series also uncovered a pre-existing UB instance thanks to an `objtool` warning which I noticed while testing the series (details in the mailing list). - Miguel ] Link: https://github.com/llvm/llvm-project/pull/170397 [1] Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Co-developed-by: Matthew Maurer Signed-off-by: Matthew Maurer Signed-off-by: Gary Guo Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl Reviewed-by: Nathan Chancellor Tested-by: Nathan Chancellor Tested-by: Andreas Hindborg Reviewed-by: Andreas Hindborg Link: https://patch.msgid.link/20260203-inline-helpers-v2-3-beb8547a03c9@google.com [ Some changes, apart from the rebase: - Added "(EXPERIMENTAL)" to Kconfig as the commit mentions. - Added `depends on ARM64 || X86_64` and `!UML` for now, since this is experimental, other architectures may require other changes (e.g. the issues I mentioned in the mailing list for ARM and UML) and they are not really tested so far. So let arch maintainers pick this up if they think it is worth it. - Gated the `cmd_ld_single` step also into the new mode, which also means that any possible future `objcopy` step is done after the translation, as expected. - Added `.gitignore` for `.bc` with exception for existing script. - Added `part-of-*` for helpers bitcode files as discussed, and dropped `$(if $(filter %_module.bc,$@),-DMODULE)` since `-DMODULE` is already there (would be duplicated otherwise). - Moved `LLVM_LINK` to keep binutils list alphabetized. - Fixed typo in title. - Dropped second `cmd_ld_single` commit message paragraph. - Miguel ] Signed-off-by: Miguel Ojeda --- diff --git a/.gitignore b/.gitignore index 3a7241c941f5e..3044b9590f058 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ .* *.a *.asn1.[ch] +*.bc *.bin *.bz2 *.c.[012]*.* @@ -184,3 +185,6 @@ sphinx_*/ # Rust analyzer configuration /rust-project.json + +# bc language scripts (not LLVM bitcode) +!kernel/time/timeconst.bc diff --git a/Makefile b/Makefile index 2b15f0b4a0cb5..1a219bf1c7718 100644 --- a/Makefile +++ b/Makefile @@ -515,6 +515,7 @@ ifneq ($(LLVM),) CC = $(LLVM_PREFIX)clang$(LLVM_SUFFIX) LD = $(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX) AR = $(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX) +LLVM_LINK = $(LLVM_PREFIX)llvm-link$(LLVM_SUFFIX) NM = $(LLVM_PREFIX)llvm-nm$(LLVM_SUFFIX) OBJCOPY = $(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX) OBJDUMP = $(LLVM_PREFIX)llvm-objdump$(LLVM_SUFFIX) @@ -628,7 +629,7 @@ export RUSTC_BOOTSTRAP := 1 export CLIPPY_CONF_DIR := $(srctree) export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG -export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN +export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN LLVM_LINK export HOSTRUSTC KBUILD_HOSTRUSTFLAGS export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 93f356d2b3d95..30d0696763099 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -3577,6 +3577,23 @@ config RUST_KERNEL_DOCTESTS If unsure, say N. +config RUST_INLINE_HELPERS + bool "Inline C helpers into Rust code (EXPERIMENTAL)" + depends on RUST && RUSTC_CLANG_LLVM_COMPATIBLE + depends on EXPERT + depends on ARM64 || X86_64 + depends on !UML + help + Inlines C helpers into Rust code using Link Time Optimization. + + If this option is enabled, C helper functions declared in + rust/helpers/ are inlined into Rust code, which is helpful for + performance of Rust code. This requires a matching LLVM version for + Clang and rustc. + + If you are sure that you're using Clang and rustc with matching LLVM + versions, say Y. Otherwise say N. + endmenu # "Rust" endmenu # Kernel hacking diff --git a/rust/Makefile b/rust/Makefile index 629b3bdd2b206..5eca6a817966d 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -6,15 +6,19 @@ rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o always-$(CONFIG_RUST) += exports_core_generated.h +ifdef CONFIG_RUST_INLINE_HELPERS +always-$(CONFIG_RUST) += helpers/helpers.bc helpers/helpers_module.bc +else +obj-$(CONFIG_RUST) += helpers/helpers.o +always-$(CONFIG_RUST) += exports_helpers_generated.h +endif # Missing prototypes are expected in the helpers since these are exported # for Rust only, thus there is no header nor prototypes. -obj-$(CONFIG_RUST) += helpers/helpers.o CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o -always-$(CONFIG_RUST) += exports_helpers_generated.h \ - exports_bindings_generated.h exports_kernel_generated.h +always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs obj-$(CONFIG_RUST) += uapi.o @@ -495,6 +499,16 @@ $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; $(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE $(call if_changed_dep,bindgen) +quiet_cmd_rust_helper = HELPER $@ + cmd_rust_helper = \ + $(CC) $(filter-out $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) \ + -c -g0 $< -emit-llvm -o $@ + +$(obj)/helpers/helpers.bc: private part-of-builtin := y +$(obj)/helpers/helpers_module.bc: private part-of-module := y +$(obj)/helpers/helpers.bc $(obj)/helpers/helpers_module.bc: $(src)/helpers/helpers.c FORCE + +$(call if_changed_dep,rust_helper) + rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }' quiet_cmd_exports = EXPORTS $@ @@ -577,12 +591,16 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L OBJTREE=$(abspath $(objtree)) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ - --emit=dep-info=$(depfile) --emit=obj=$@ \ + --emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ -Zunstable-options \ + $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ + $(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(cmd_ld_single)) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) @@ -712,4 +730,9 @@ $(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generate endif endif +ifdef CONFIG_RUST_INLINE_HELPERS +$(obj)/kernel.o: private link_helper = 1 +$(obj)/kernel.o: $(obj)/helpers/helpers.bc +endif + endif # CONFIG_RUST diff --git a/rust/exports.c b/rust/exports.c index 587f0e776aba5..1b52460b0f4ee 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -16,10 +16,13 @@ #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) #include "exports_core_generated.h" -#include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" +#ifndef CONFIG_RUST_INLINE_HELPERS +#include "exports_helpers_generated.h" +#endif + // For modules using `rust/build_error.rs`. #ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW EXPORT_SYMBOL_RUST_GPL(rust_build_error); diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 0328afd5ee968..a6d1a2b210aaa 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -346,7 +346,12 @@ rust_common_cmd = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(cmd_objtool) + cmd_rustc_o_rs = $(rust_common_cmd) --emit=$(if $(CONFIG_RUST_INLINE_HELPERS),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) $< \ + $(if $(CONFIG_RUST_INLINE_HELPERS),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ + $(objtree)/rust/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(cmd_ld_single)) \ + $(cmd_objtool) define rule_rustc_o_rs $(call cmd_and_fixdep,rustc_o_rs)