From: Yao Zihong Date: Fri, 19 Dec 2025 23:46:42 +0000 (-0600) Subject: riscv: Add RVV memset for both multiarch and non-multiarch builds X-Git-Tag: glibc-2.43~66 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0b8a996f44b5f4c02991f02cd12bf05b17db4576;p=thirdparty%2Fglibc.git riscv: Add RVV memset for both multiarch and non-multiarch builds This patch adds an RVV-optimized implementation of memset for RISC-V and enables it for both multiarch (IFUNC) and non-multiarch builds. The implementation integrates Hau Hsu's 2023 RVV work under a unified ifunc-based framework. A vectorized version (__memset_vector) is added alongside the generic fallback (__memset_generic). The runtime resolver selects the RVV variant when RISCV_HWPROBE_KEY_IMA_EXT_0 reports vector support (RVV). Currently, the resolver still selects the RVV variant even when the RVV extension is disabled via prctl(). As a consequence, any process that has RVV disabled via prctl() will receive SIGILL when calling memset(). Co-authored-by: Jerry Shih Co-authored-by: Jeff Law Signed-off-by: Yao Zihong Reviewed-by: Peter Bergner --- diff --git a/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h new file mode 100644 index 0000000000..2ff7369678 --- /dev/null +++ b/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h @@ -0,0 +1,26 @@ +/* Symbol redirection for loader/static initialization code. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#ifndef _DL_IFUNC_GENERIC_H +#define _DL_IFUNC_GENERIC_H + +#ifndef SHARED +asm ("memset = __memset_generic"); +#endif + +#endif diff --git a/sysdeps/riscv/multiarch/memset-generic.c b/sysdeps/riscv/multiarch/memset-generic.c new file mode 100644 index 0000000000..c93bb43c8f --- /dev/null +++ b/sysdeps/riscv/multiarch/memset-generic.c @@ -0,0 +1,26 @@ +/* Re-include the default memset implementation. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#if IS_IN(libc) +# define MEMSET __memset_generic +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(x) +#endif +#include diff --git a/sysdeps/riscv/multiarch/memset-vector.S b/sysdeps/riscv/multiarch/memset-vector.S new file mode 100644 index 0000000000..83d8c2011e --- /dev/null +++ b/sysdeps/riscv/multiarch/memset-vector.S @@ -0,0 +1,25 @@ + +/* RISC-V RVV based memset. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if IS_IN(libc) +# define MEMSET __memset_vector +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(name) +#include +#endif diff --git a/sysdeps/riscv/preconfigure b/sysdeps/riscv/preconfigure index a96cd0d7f8..92ffa1ac02 100644 --- a/sysdeps/riscv/preconfigure +++ b/sysdeps/riscv/preconfigure @@ -60,6 +60,7 @@ riscv*) version=`$CC $CFLAGS $CPPFLAGS -E -dM -xc /dev/null | sed -n 's/^#define __GNUC__ \(.*\)/\1/p'` test $version -lt 15 && as_fn_error 1 "glibc requires GCC 15 or later for the V extension" "$LINENO" 5 test $vector -lt "1000000" && as_fn_error 1 "glibc requires at least RVV 1.0 for the V extension" "$LINENO" 5 + float_machine=rvv fi base_machine=riscv diff --git a/sysdeps/riscv/preconfigure.ac b/sysdeps/riscv/preconfigure.ac index f95ffe83fb..99fbb0c9c9 100644 --- a/sysdeps/riscv/preconfigure.ac +++ b/sysdeps/riscv/preconfigure.ac @@ -60,6 +60,7 @@ riscv*) version=`$CC $CFLAGS $CPPFLAGS -E -dM -xc /dev/null | sed -n 's/^#define __GNUC__ \(.*\)/\1/p'` test $version -lt 15 && AC_MSG_ERROR([glibc requires GCC 15 or later for the V extension], [1]) test $vector -lt "1000000" && AC_MSG_ERROR([glibc requires at least RVV 1.0 for the V extension], [1]) + float_machine=rvv fi base_machine=riscv diff --git a/sysdeps/riscv/rv32/rvv/Implies b/sysdeps/riscv/rv32/rvv/Implies new file mode 100644 index 0000000000..25ce1df222 --- /dev/null +++ b/sysdeps/riscv/rv32/rvv/Implies @@ -0,0 +1,2 @@ +riscv/rv32/rvd +riscv/rvv diff --git a/sysdeps/riscv/rv64/rvv/Implies b/sysdeps/riscv/rv64/rvv/Implies new file mode 100644 index 0000000000..9993bb30e3 --- /dev/null +++ b/sysdeps/riscv/rv64/rvv/Implies @@ -0,0 +1,2 @@ +riscv/rv64/rvd +riscv/rvv diff --git a/sysdeps/riscv/rvv/memset.S b/sysdeps/riscv/rvv/memset.S new file mode 100644 index 0000000000..2b81a937f5 --- /dev/null +++ b/sysdeps/riscv/rvv/memset.S @@ -0,0 +1,53 @@ +/* RISC-V RVV based memset. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include + +#ifndef MEMSET +# define MEMSET memset +#endif + +#define dst a0 +#define value a1 +#define num a2 + +#define ivl a3 +#define dst_ptr a5 + +#define ELEM_LMUL_SETTING m8 +#define vdata v0 + +ENTRY (MEMSET) +.option push +.option arch, +v + mv dst_ptr, dst + + vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma + vmv.v.x vdata, value +L(loop): + vse8.v vdata, (dst_ptr) + sub num, num, ivl + add dst_ptr, dst_ptr, ivl + vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma + bnez num, L(loop) + + ret +.option pop +END (MEMSET) +libc_hidden_builtin_def (memset) diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile index fcef5659d4..1d26966ded 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile @@ -3,6 +3,9 @@ sysdep_routines += \ memcpy \ memcpy-generic \ memcpy_noalignment \ + memset \ + memset-generic \ + memset-vector \ # sysdep_routines CFLAGS-memcpy_noalignment.c += -mno-strict-align diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c index 1c1deca8f6..87456f3370 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c @@ -27,17 +27,31 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, size_t i = max; bool fast_unaligned = false; + bool rvv_enabled = false; - struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_CPUPERF_0 }; - if (__riscv_hwprobe (&pair, 1, 0, NULL, 0) == 0 - && (pair.value & RISCV_HWPROBE_MISALIGNED_MASK) + struct riscv_hwprobe pairs[2] = { + {.key = RISCV_HWPROBE_KEY_CPUPERF_0}, + {.key = RISCV_HWPROBE_KEY_IMA_EXT_0} + }; + + if (__riscv_hwprobe (pairs, 2, 0, NULL, 0) == 0) { + if ((pairs[0].value & RISCV_HWPROBE_MISALIGNED_MASK) == RISCV_HWPROBE_MISALIGNED_FAST) - fast_unaligned = true; + fast_unaligned = true; + + if (pairs[1].value & RISCV_HWPROBE_IMA_V) + rvv_enabled = true; + } IFUNC_IMPL (i, name, memcpy, IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned, __memcpy_noalignment) IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic)) + IFUNC_IMPL (i, name, memset, + IFUNC_IMPL_ADD (array, i, memset, rvv_enabled, + __memset_vector) + IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic)) + return 0; } diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memset.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memset.c new file mode 100644 index 0000000000..166427bff8 --- /dev/null +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memset.c @@ -0,0 +1,58 @@ +/* Multiple versions of memset. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if IS_IN (libc) +/* Redefine memset so that the compiler won't complain about the type + mismatch with the IFUNC selector in strong_alias, below. */ +# undef memset +# define memset __redirect_memset +# include +# include +# include +# include +# include + +extern __typeof (__redirect_memset) __libc_memset; + +extern __typeof (__redirect_memset) __memset_generic attribute_hidden; +extern __typeof (__redirect_memset) __memset_vector attribute_hidden; + +static inline __typeof (__redirect_memset) * +select_memset_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func) +{ + unsigned long long v; + + if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0 + && (v & RISCV_HWPROBE_IMA_V) == RISCV_HWPROBE_IMA_V) + return __memset_vector; + + return __memset_generic; +} + +riscv_libc_ifunc (__libc_memset, select_memset_ifunc); + +# undef memset +strong_alias (__libc_memset, memset); +# ifdef SHARED +__hidden_ver1 (memset, __GI_memset, __redirect_memset) + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memset); +# endif +#else +# include +#endif