From: Yao Zihong Date: Sun, 21 Jun 2026 03:54:22 +0000 (-0500) Subject: riscv: Add RVV memmove for both multiarch and non-multiarch builds X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;p=thirdparty%2Fglibc.git riscv: Add RVV memmove for both multiarch and non-multiarch builds This patch adds an RVV-optimized implementation of memmove for RISC-V and enables it for both multiarch (IFUNC) and non-multiarch builds. The implementation integrates Hau Hsu's 2023 RVV work under a unified ifunc-based framework. A vectorized version (__memmove_vector) is added alongside the generic fallback (__memmove_generic). The runtime resolver selects the RVV variant when RISCV_HWPROBE_KEY_IMA_EXT_0 reports vector support (RVV). Currently, the resolver still selects the RVV variant even when the RVV extension is disabled via prctl(). As a consequence, any process that has RVV disabled via prctl() will receive SIGILL when calling memmove(). Co-authored-by: Hau Hsu Co-authored-by: Jerry Shih Signed-off-by: Yao Zihong Reviewed-by: Peter Bergner --- diff --git a/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h index f81e5b7e6d..852d3f1c14 100644 --- a/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h +++ b/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h @@ -22,6 +22,7 @@ #ifndef SHARED asm ("memcmp = __memcmp_generic"); asm ("memcpy = __memcpy_generic"); +asm ("memmove = __memmove_generic"); asm ("memset = __memset_generic"); asm ("strlen = __strlen_generic"); #endif diff --git a/sysdeps/riscv/multiarch/memmove-generic.c b/sysdeps/riscv/multiarch/memmove-generic.c new file mode 100644 index 0000000000..44c01c887a --- /dev/null +++ b/sysdeps/riscv/multiarch/memmove-generic.c @@ -0,0 +1,26 @@ +/* Re-include the default memmove implementation. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#if IS_IN(libc) +# define MEMMOVE __memmove_generic +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(x) +# include +#endif diff --git a/sysdeps/riscv/multiarch/memmove-vector.S b/sysdeps/riscv/multiarch/memmove-vector.S new file mode 100644 index 0000000000..6bf2d9f614 --- /dev/null +++ b/sysdeps/riscv/multiarch/memmove-vector.S @@ -0,0 +1,24 @@ +/* Re-include the RISC-V RVV based memmove implementation. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if IS_IN(libc) +# define MEMMOVE __memmove_vector +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(name) +# include +#endif diff --git a/sysdeps/riscv/rvv/memmove.S b/sysdeps/riscv/rvv/memmove.S new file mode 100644 index 0000000000..71ebb6aeb1 --- /dev/null +++ b/sysdeps/riscv/rvv/memmove.S @@ -0,0 +1,79 @@ +/* RISC-V RVV based memmove. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include + +#ifndef MEMMOVE +# define MEMMOVE memmove +#endif + +#define dst a0 +#define src a1 +#define num a2 + +#define ivl a3 +#define dst_ptr a4 +#define diff a5 + +#define ELEM_LMUL_SETTING m8 +#define vdata v0 + +ENTRY (MEMMOVE) +.option push +.option arch, +v + /* There is nothing to do if we are copying zero bytes or DST == SRC. */ + beqz num, L(done) + sub diff, dst, src + beqz diff, L(done) + + /* If (unsigned)(dst - src) < num, then we have an overlap scenario that + requires a backward copy loop. Otherwise, we can use a faster forward + copy loop. */ + bltu diff, num, L(do_backward_loop) + + mv dst_ptr, dst + +L(forward_copy_loop): + vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma + vle8.v vdata, (src) + sub num, num, ivl + add src, src, ivl + vse8.v vdata, (dst_ptr) + add dst_ptr, dst_ptr, ivl + bnez num, L(forward_copy_loop) +L(done): + ret + +L(do_backward_loop): + add src, src, num + add dst_ptr, dst, num + +L(backward_copy_loop): + vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma + sub src, src, ivl + sub dst_ptr, dst_ptr, ivl + vle8.v vdata, (src) + sub num, num, ivl + vse8.v vdata, (dst_ptr) + bnez num, L(backward_copy_loop) + ret + +.option pop +END (MEMMOVE) +libc_hidden_builtin_def (memmove) diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile index 30d92c1ba9..929df14a6f 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile @@ -13,6 +13,9 @@ sysdep_routines += \ memcpy-generic \ memcpy-vector \ memcpy_noalignment \ + memmove \ + memmove-generic \ + memmove-vector \ memset \ memset-generic \ memset-vector \ diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c index 8027578529..4c28e22606 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c @@ -105,5 +105,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __strrchr_vector) IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_generic)) + IFUNC_IMPL (i, name, memmove, + IFUNC_IMPL_ADD (array, i, memmove, rvv_enabled, + __memmove_vector) + IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_generic)) + return 0; } diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memmove.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memmove.c new file mode 100644 index 0000000000..670f108029 --- /dev/null +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memmove.c @@ -0,0 +1,56 @@ +/* Multiple versions of memmove. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if IS_IN (libc) +/* Redefine memmove so that the compiler won't complain about the type + mismatch with the IFUNC selector in strong_alias, below. */ +# undef memmove +# define memmove __redirect_memmove +# include +# include +# include +# include +# include + +extern __typeof (__redirect_memmove) __libc_memmove; + +extern __typeof (__redirect_memmove) __memmove_generic attribute_hidden; +extern __typeof (__redirect_memmove) __memmove_vector attribute_hidden; + +static inline __typeof (__redirect_memmove) * +select_memmove_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func) +{ + unsigned long long int v; + if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0 + && (v & RISCV_HWPROBE_IMA_V) == RISCV_HWPROBE_IMA_V) + return __memmove_vector; + return __memmove_generic; +} + +riscv_libc_ifunc (__libc_memmove, select_memmove_ifunc); + +# undef memmove +strong_alias (__libc_memmove, memmove); +# ifdef SHARED +__hidden_ver1 (memmove, __GI_memmove, __redirect_memmove) + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memmove); +# endif +#else +# include +#endif