From: Yao Zihong Date: Fri, 5 Jun 2026 00:25:00 +0000 (-0500) Subject: riscv: Add RVV memccpy for both multiarch and non-multiarch builds X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=496ccfe31239c09fb4ef1efbd084905e2dbb3fea;p=thirdparty%2Fglibc.git riscv: Add RVV memccpy for both multiarch and non-multiarch builds This patch adds an RVV-optimized implementation of memccpy for RISC-V and enables it for both multiarch (IFUNC) and non-multiarch builds. The implementation integrates Hau Hsu's 2023 RVV work under a unified ifunc-based framework. A vectorized version (__memccpy_vector) is added alongside the generic fallback (__memccpy_generic). The runtime resolver selects the RVV variant when RISCV_HWPROBE_KEY_IMA_EXT_0 reports vector support (RVV). Currently, the resolver still selects the RVV variant even when the RVV extension is disabled via prctl(). As a consequence, any process that has RVV disabled via prctl() will receive SIGILL when calling memccpy(). Co-authored-by: Hau Hsu Co-authored-by: Jerry Shih Signed-off-by: Yao Zihong Reviewed-by: Peter Bergner --- diff --git a/sysdeps/riscv/multiarch/memccpy-generic.c b/sysdeps/riscv/multiarch/memccpy-generic.c new file mode 100644 index 0000000000..d7e6e2d847 --- /dev/null +++ b/sysdeps/riscv/multiarch/memccpy-generic.c @@ -0,0 +1,26 @@ +/* Re-include the default memccpy implementation. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#if IS_IN(libc) +# define MEMCCPY __memccpy_generic +# undef weak_alias +# define weak_alias(x, x2) +# include +#endif diff --git a/sysdeps/riscv/multiarch/memccpy-vector.S b/sysdeps/riscv/multiarch/memccpy-vector.S new file mode 100644 index 0000000000..8ea121e444 --- /dev/null +++ b/sysdeps/riscv/multiarch/memccpy-vector.S @@ -0,0 +1,24 @@ +/* Re-include the RISC-V RVV based memccpy implementation. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if IS_IN(libc) +# define MEMCCPY __memccpy_vector +# undef weak_alias +# define weak_alias(name, alias) +# include +#endif diff --git a/sysdeps/riscv/rvv/memccpy.S b/sysdeps/riscv/rvv/memccpy.S new file mode 100644 index 0000000000..5854d463c2 --- /dev/null +++ b/sysdeps/riscv/rvv/memccpy.S @@ -0,0 +1,67 @@ +/* RISC-V RVV based memccpy. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include + +#ifndef MEMCCPY +# define MEMCCPY __memccpy +#endif + +#define dst a0 +#define src a1 +#define c_val a2 +#define num a3 + +#define ivl a4 +#define dst_ptr a5 +#define vfirst_idx a6 + +#define ELEM_LMUL_SETTING m8 +#define vdata v0 +#define vmask v8 + +ENTRY (MEMCCPY) +.option push +.option arch, +v + mv dst_ptr, dst +L(loop): + vsetvli zero, num, e8, ELEM_LMUL_SETTING, ta, ma + vle8ff.v vdata, (src) + + vmseq.vx vmask, vdata, c_val + vfirst.m vfirst_idx, vmask + csrr ivl, vl + bgez vfirst_idx, L(found) + vse8.v vdata, (dst_ptr) + + sub num, num, ivl + add src, src, ivl + add dst_ptr, dst_ptr, ivl + bnez num, L(loop) + mv a0, zero + ret +L(found): + addi vfirst_idx, vfirst_idx, 1 + vsetvli zero, vfirst_idx, e8, ELEM_LMUL_SETTING, ta, ma + vse8.v vdata, (dst_ptr) + add a0, dst_ptr, vfirst_idx + ret +.option pop +END (MEMCCPY) +weak_alias (MEMCCPY, memccpy) diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile index 20ff2bdca2..52730cf01c 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile @@ -1,5 +1,8 @@ ifeq ($(subdir),string) sysdep_routines += \ + memccpy \ + memccpy-generic \ + memccpy-vector \ memcmp \ memcmp-generic \ memcmp-vector \ diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c index 1d09cf27a3..a373de6c89 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c @@ -80,6 +80,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __strncmp_vector) IFUNC_IMPL_ADD (array, i, strncmp, 1, __strncmp_generic)) + IFUNC_IMPL (i, name, memccpy, + IFUNC_IMPL_ADD (array, i, memccpy, rvv_enabled, + __memccpy_vector) + IFUNC_IMPL_ADD (array, i, memccpy, 1, __memccpy_generic)) + IFUNC_IMPL (i, name, memcmp, IFUNC_IMPL_ADD (array, i, memcmp, rvv_enabled, __memcmp_vector) diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memccpy.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memccpy.c new file mode 100644 index 0000000000..84d300b3cc --- /dev/null +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memccpy.c @@ -0,0 +1,58 @@ +/* Multiple versions of memccpy. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if IS_IN (libc) +/* Redefine memccpy so that the compiler won't complain about the type + mismatch with the IFUNC selector in strong_alias, below. */ +# undef memccpy +# define memccpy __redirect_memccpy +# include +# include +# include +# include +# include + +extern __typeof (__redirect_memccpy) __libc_memccpy; + +extern __typeof (__redirect_memccpy) __memccpy_generic attribute_hidden; +extern __typeof (__redirect_memccpy) __memccpy_vector attribute_hidden; + +static inline __typeof (__redirect_memccpy) * +select_memccpy_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func) +{ + unsigned long long v; + + if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0 + && (v & RISCV_HWPROBE_IMA_V) == RISCV_HWPROBE_IMA_V) + return __memccpy_vector; + + return __memccpy_generic; +} + +riscv_libc_ifunc (__libc_memccpy, select_memccpy_ifunc); + +# undef memccpy +weak_alias (__libc_memccpy, memccpy); +# ifdef SHARED +__hidden_ver1 (memccpy, __GI_memccpy, __redirect_memccpy) + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memccpy); +# endif +#else +# include +#endif