+++ /dev/null
-From 9566c2298b24e0c594a2f5402474ecb3137134e6 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Wed, 23 Jun 2021 21:40:39 +0900
-Subject: riscv: __asm_copy_to-from_user: Optimize unaligned memory access and
- pipeline stall
-
-From: Akira Tsukamoto <akira.tsukamoto@gmail.com>
-
-[ Upstream commit ca6eaaa210deec0e41cbfc380bf89cf079203569 ]
-
-This patch will reduce cpu usage dramatically in kernel space especially
-for application which use sys-call with large buffer size, such as
-network applications. The main reason behind this is that every
-unaligned memory access will raise exceptions and switch between s-mode
-and m-mode causing large overhead.
-
-First copy in bytes until reaches the first word aligned boundary in
-destination memory address. This is the preparation before the bulk
-aligned word copy.
-
-The destination address is aligned now, but oftentimes the source
-address is not in an aligned boundary. To reduce the unaligned memory
-access, it reads the data from source in aligned boundaries, which will
-cause the data to have an offset, and then combines the data in the next
-iteration by fixing offset with shifting before writing to destination.
-The majority of the improving copy speed comes from this shift copy.
-
-In the lucky situation that the both source and destination address are
-on the aligned boundary, perform load and store with register size to
-copy the data. Without the unrolling, it will reduce the speed since the
-next store instruction for the same register using from the load will
-stall the pipeline.
-
-At last, copying the remainder in one byte at a time.
-
-Signed-off-by: Akira Tsukamoto <akira.tsukamoto@gmail.com>
-Signed-off-by: Palmer Dabbelt <palmerdabbelt@google.com>
-Stable-dep-of: 4b05b993900d ("riscv: uaccess: Return the number of bytes effectively not copied")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/riscv/lib/uaccess.S | 181 +++++++++++++++++++++++++++++++--------
- 1 file changed, 146 insertions(+), 35 deletions(-)
-
-diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S
-index fceaeb18cc640..bceb0629e440e 100644
---- a/arch/riscv/lib/uaccess.S
-+++ b/arch/riscv/lib/uaccess.S
-@@ -19,50 +19,161 @@ ENTRY(__asm_copy_from_user)
- li t6, SR_SUM
- csrs CSR_STATUS, t6
-
-- add a3, a1, a2
-- /* Use word-oriented copy only if low-order bits match */
-- andi t0, a0, SZREG-1
-- andi t1, a1, SZREG-1
-- bne t0, t1, 2f
-+ /* Save for return value */
-+ mv t5, a2
-
-- addi t0, a1, SZREG-1
-- andi t1, a3, ~(SZREG-1)
-- andi t0, t0, ~(SZREG-1)
- /*
-- * a3: terminal address of source region
-- * t0: lowest XLEN-aligned address in source
-- * t1: highest XLEN-aligned address in source
-+ * Register allocation for code below:
-+ * a0 - start of uncopied dst
-+ * a1 - start of uncopied src
-+ * a2 - size
-+ * t0 - end of uncopied dst
- */
-- bgeu t0, t1, 2f
-- bltu a1, t0, 4f
-+ add t0, a0, a2
-+ bgtu a0, t0, 5f
-+
-+ /*
-+ * Use byte copy only if too small.
-+ */
-+ li a3, 8*SZREG /* size must be larger than size in word_copy */
-+ bltu a2, a3, .Lbyte_copy_tail
-+
-+ /*
-+ * Copy first bytes until dst is align to word boundary.
-+ * a0 - start of dst
-+ * t1 - start of aligned dst
-+ */
-+ addi t1, a0, SZREG-1
-+ andi t1, t1, ~(SZREG-1)
-+ /* dst is already aligned, skip */
-+ beq a0, t1, .Lskip_first_bytes
- 1:
-- fixup REG_L, t2, (a1), 10f
-- fixup REG_S, t2, (a0), 10f
-- addi a1, a1, SZREG
-- addi a0, a0, SZREG
-- bltu a1, t1, 1b
-+ /* a5 - one byte for copying data */
-+ fixup lb a5, 0(a1), 10f
-+ addi a1, a1, 1 /* src */
-+ fixup sb a5, 0(a0), 10f
-+ addi a0, a0, 1 /* dst */
-+ bltu a0, t1, 1b /* t1 - start of aligned dst */
-+
-+.Lskip_first_bytes:
-+ /*
-+ * Now dst is aligned.
-+ * Use shift-copy if src is misaligned.
-+ * Use word-copy if both src and dst are aligned because
-+ * can not use shift-copy which do not require shifting
-+ */
-+ /* a1 - start of src */
-+ andi a3, a1, SZREG-1
-+ bnez a3, .Lshift_copy
-+
-+.Lword_copy:
-+ /*
-+ * Both src and dst are aligned, unrolled word copy
-+ *
-+ * a0 - start of aligned dst
-+ * a1 - start of aligned src
-+ * a3 - a1 & mask:(SZREG-1)
-+ * t0 - end of aligned dst
-+ */
-+ addi t0, t0, -(8*SZREG-1) /* not to over run */
- 2:
-- bltu a1, a3, 5f
-+ fixup REG_L a4, 0(a1), 10f
-+ fixup REG_L a5, SZREG(a1), 10f
-+ fixup REG_L a6, 2*SZREG(a1), 10f
-+ fixup REG_L a7, 3*SZREG(a1), 10f
-+ fixup REG_L t1, 4*SZREG(a1), 10f
-+ fixup REG_L t2, 5*SZREG(a1), 10f
-+ fixup REG_L t3, 6*SZREG(a1), 10f
-+ fixup REG_L t4, 7*SZREG(a1), 10f
-+ fixup REG_S a4, 0(a0), 10f
-+ fixup REG_S a5, SZREG(a0), 10f
-+ fixup REG_S a6, 2*SZREG(a0), 10f
-+ fixup REG_S a7, 3*SZREG(a0), 10f
-+ fixup REG_S t1, 4*SZREG(a0), 10f
-+ fixup REG_S t2, 5*SZREG(a0), 10f
-+ fixup REG_S t3, 6*SZREG(a0), 10f
-+ fixup REG_S t4, 7*SZREG(a0), 10f
-+ addi a0, a0, 8*SZREG
-+ addi a1, a1, 8*SZREG
-+ bltu a0, t0, 2b
-+
-+ addi t0, t0, 8*SZREG-1 /* revert to original value */
-+ j .Lbyte_copy_tail
-+
-+.Lshift_copy:
-+
-+ /*
-+ * Word copy with shifting.
-+ * For misaligned copy we still perform aligned word copy, but
-+ * we need to use the value fetched from the previous iteration and
-+ * do some shifts.
-+ * This is safe because reading less than a word size.
-+ *
-+ * a0 - start of aligned dst
-+ * a1 - start of src
-+ * a3 - a1 & mask:(SZREG-1)
-+ * t0 - end of uncopied dst
-+ * t1 - end of aligned dst
-+ */
-+ /* calculating aligned word boundary for dst */
-+ andi t1, t0, ~(SZREG-1)
-+ /* Converting unaligned src to aligned arc */
-+ andi a1, a1, ~(SZREG-1)
-+
-+ /*
-+ * Calculate shifts
-+ * t3 - prev shift
-+ * t4 - current shift
-+ */
-+ slli t3, a3, LGREG
-+ li a5, SZREG*8
-+ sub t4, a5, t3
-+
-+ /* Load the first word to combine with seceond word */
-+ fixup REG_L a5, 0(a1), 10f
-
- 3:
-+ /* Main shifting copy
-+ *
-+ * a0 - start of aligned dst
-+ * a1 - start of aligned src
-+ * t1 - end of aligned dst
-+ */
-+
-+ /* At least one iteration will be executed */
-+ srl a4, a5, t3
-+ fixup REG_L a5, SZREG(a1), 10f
-+ addi a1, a1, SZREG
-+ sll a2, a5, t4
-+ or a2, a2, a4
-+ fixup REG_S a2, 0(a0), 10f
-+ addi a0, a0, SZREG
-+ bltu a0, t1, 3b
-+
-+ /* Revert src to original unaligned value */
-+ add a1, a1, a3
-+
-+.Lbyte_copy_tail:
-+ /*
-+ * Byte copy anything left.
-+ *
-+ * a0 - start of remaining dst
-+ * a1 - start of remaining src
-+ * t0 - end of remaining dst
-+ */
-+ bgeu a0, t0, 5f
-+4:
-+ fixup lb a5, 0(a1), 10f
-+ addi a1, a1, 1 /* src */
-+ fixup sb a5, 0(a0), 10f
-+ addi a0, a0, 1 /* dst */
-+ bltu a0, t0, 4b /* t0 - end of dst */
-+
-+5:
- /* Disable access to user memory */
- csrc CSR_STATUS, t6
-- li a0, 0
-+ li a0, 0
- ret
--4: /* Edge case: unalignment */
-- fixup lbu, t2, (a1), 10f
-- fixup sb, t2, (a0), 10f
-- addi a1, a1, 1
-- addi a0, a0, 1
-- bltu a1, t0, 4b
-- j 1b
--5: /* Edge case: remainder */
-- fixup lbu, t2, (a1), 10f
-- fixup sb, t2, (a0), 10f
-- addi a1, a1, 1
-- addi a0, a0, 1
-- bltu a1, a3, 5b
-- j 3b
- ENDPROC(__asm_copy_to_user)
- ENDPROC(__asm_copy_from_user)
- EXPORT_SYMBOL(__asm_copy_to_user)
-@@ -117,7 +228,7 @@ EXPORT_SYMBOL(__clear_user)
- 10:
- /* Disable access to user memory */
- csrs CSR_STATUS, t6
-- mv a0, a2
-+ mv a0, t5
- ret
- 11:
- csrs CSR_STATUS, t6
---
-2.40.1
-
+++ /dev/null
-From e1e850d6f778aa4f1a5485647020999e39e6483e Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Wed, 15 Jun 2022 09:47:14 +0800
-Subject: riscv: lib: uaccess: fix CSR_STATUS SR_SUM bit
-
-From: Chen Lifu <chenlifu@huawei.com>
-
-[ Upstream commit c08b4848f596fd95543197463b5162bd7bab2442 ]
-
-Since commit 5d8544e2d007 ("RISC-V: Generic library routines and assembly")
-and commit ebcbd75e3962 ("riscv: Fix the bug in memory access fixup code"),
-if __clear_user and __copy_user return from an fixup branch,
-CSR_STATUS SR_SUM bit will be set, it is a vulnerability, so that
-S-mode memory accesses to pages that are accessible by U-mode will success.
-Disable S-mode access to U-mode memory should clear SR_SUM bit.
-
-Fixes: 5d8544e2d007 ("RISC-V: Generic library routines and assembly")
-Fixes: ebcbd75e3962 ("riscv: Fix the bug in memory access fixup code")
-Signed-off-by: Chen Lifu <chenlifu@huawei.com>
-Reviewed-by: Ben Dooks <ben.dooks@codethink.co.uk>
-Link: https://lore.kernel.org/r/20220615014714.1650349-1-chenlifu@huawei.com
-Cc: stable@vger.kernel.org
-Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
-Stable-dep-of: 4b05b993900d ("riscv: uaccess: Return the number of bytes effectively not copied")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/riscv/lib/uaccess.S | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S
-index baddd6a0d0229..039050172d083 100644
---- a/arch/riscv/lib/uaccess.S
-+++ b/arch/riscv/lib/uaccess.S
-@@ -178,7 +178,7 @@ ENTRY(__asm_copy_from_user)
- /* Exception fixup code */
- 10:
- /* Disable access to user memory */
-- csrs CSR_STATUS, t6
-+ csrc CSR_STATUS, t6
- mv a0, t5
- ret
- ENDPROC(__asm_copy_to_user)
-@@ -230,7 +230,7 @@ ENTRY(__clear_user)
- /* Exception fixup code */
- 11:
- /* Disable access to user memory */
-- csrs CSR_STATUS, t6
-+ csrc CSR_STATUS, t6
- mv a0, a1
- ret
- ENDPROC(__clear_user)
---
-2.40.1
-
+++ /dev/null
-From 525c66da537d4229207c1ed8365e0f5c834844b5 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Thu, 18 Nov 2021 19:25:14 +0800
-Subject: riscv: lib: uaccess: fold fixups into body
-
-From: Jisheng Zhang <jszhang@kernel.org>
-
-[ Upstream commit 9d504f9aa5c1b76673018da9503e76b351a24b8c ]
-
-uaccess functions such __asm_copy_to_user(), __arch_copy_from_user()
-and __clear_user() place their exception fixups in the `.fixup` section
-without any clear association with themselves. If we backtrace the
-fixup code, it will be symbolized as an offset from the nearest prior
-symbol.
-
-Similar as arm64 does, we must move fixups into the body of the
-functions themselves, after the usual fast-path returns.
-
-Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
-Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
-Stable-dep-of: 4b05b993900d ("riscv: uaccess: Return the number of bytes effectively not copied")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/riscv/lib/uaccess.S | 22 +++++++++++-----------
- 1 file changed, 11 insertions(+), 11 deletions(-)
-
-diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S
-index bceb0629e440e..baddd6a0d0229 100644
---- a/arch/riscv/lib/uaccess.S
-+++ b/arch/riscv/lib/uaccess.S
-@@ -174,6 +174,13 @@ ENTRY(__asm_copy_from_user)
- csrc CSR_STATUS, t6
- li a0, 0
- ret
-+
-+ /* Exception fixup code */
-+10:
-+ /* Disable access to user memory */
-+ csrs CSR_STATUS, t6
-+ mv a0, t5
-+ ret
- ENDPROC(__asm_copy_to_user)
- ENDPROC(__asm_copy_from_user)
- EXPORT_SYMBOL(__asm_copy_to_user)
-@@ -219,19 +226,12 @@ ENTRY(__clear_user)
- addi a0, a0, 1
- bltu a0, a3, 5b
- j 3b
--ENDPROC(__clear_user)
--EXPORT_SYMBOL(__clear_user)
-
-- .section .fixup,"ax"
-- .balign 4
-- /* Fixup code for __copy_user(10) and __clear_user(11) */
--10:
-- /* Disable access to user memory */
-- csrs CSR_STATUS, t6
-- mv a0, t5
-- ret
-+ /* Exception fixup code */
- 11:
-+ /* Disable access to user memory */
- csrs CSR_STATUS, t6
- mv a0, a1
- ret
-- .previous
-+ENDPROC(__clear_user)
-+EXPORT_SYMBOL(__clear_user)
---
-2.40.1
-
+++ /dev/null
-From df655fd08a44881aa74234e48b5b24cd2bd32dd5 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Fri, 11 Aug 2023 17:06:04 +0200
-Subject: riscv: uaccess: Return the number of bytes effectively not copied
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-From: Alexandre Ghiti <alexghiti@rivosinc.com>
-
-[ Upstream commit 4b05b993900dd3eba0fc83ef5c5ddc7d65d786c6 ]
-
-It was reported that the riscv kernel hangs while executing the test
-in [1].
-
-Indeed, the test hangs when trying to write a buffer to a file. The
-problem is that the riscv implementation of raw_copy_from_user() does not
-return the correct number of bytes not written when an exception happens
-and is fixed up, instead it always returns the initial size to copy,
-even if some bytes were actually copied.
-
-generic_perform_write() pre-faults the user pages and bails out if nothing
-can be written, otherwise it will access the userspace buffer: here the
-riscv implementation keeps returning it was not able to copy any byte
-though the pre-faulting indicates otherwise. So generic_perform_write()
-keeps retrying to access the user memory and ends up in an infinite
-loop.
-
-Note that before the commit mentioned in [1] that introduced this
-regression, it worked because generic_perform_write() would bail out if
-only one byte could not be written.
-
-So fix this by returning the number of bytes effectively not written in
-__asm_copy_[to|from]_user() and __clear_user(), as it is expected.
-
-Link: https://lore.kernel.org/linux-riscv/20230309151841.bomov6hq3ybyp42a@debian/ [1]
-Fixes: ebcbd75e3962 ("riscv: Fix the bug in memory access fixup code")
-Reported-by: Bo YU <tsu.yubo@gmail.com>
-Closes: https://lore.kernel.org/linux-riscv/20230309151841.bomov6hq3ybyp42a@debian/#t
-Reported-by: Aurelien Jarno <aurelien@aurel32.net>
-Closes: https://lore.kernel.org/linux-riscv/ZNOnCakhwIeue3yr@aurel32.net/
-Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
-Reviewed-by: Björn Töpel <bjorn@rivosinc.com>
-Tested-by: Aurelien Jarno <aurelien@aurel32.net>
-Reviewed-by: Aurelien Jarno <aurelien@aurel32.net>
-Link: https://lore.kernel.org/r/20230811150604.1621784-1-alexghiti@rivosinc.com
-Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/riscv/lib/uaccess.S | 11 +++++++----
- 1 file changed, 7 insertions(+), 4 deletions(-)
-
-diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S
-index 039050172d083..80866dea07418 100644
---- a/arch/riscv/lib/uaccess.S
-+++ b/arch/riscv/lib/uaccess.S
-@@ -19,8 +19,11 @@ ENTRY(__asm_copy_from_user)
- li t6, SR_SUM
- csrs CSR_STATUS, t6
-
-- /* Save for return value */
-- mv t5, a2
-+ /*
-+ * Save the terminal address which will be used to compute the number
-+ * of bytes copied in case of a fixup exception.
-+ */
-+ add t5, a0, a2
-
- /*
- * Register allocation for code below:
-@@ -179,7 +182,7 @@ ENTRY(__asm_copy_from_user)
- 10:
- /* Disable access to user memory */
- csrc CSR_STATUS, t6
-- mv a0, t5
-+ sub a0, t5, a0
- ret
- ENDPROC(__asm_copy_to_user)
- ENDPROC(__asm_copy_from_user)
-@@ -231,7 +234,7 @@ ENTRY(__clear_user)
- 11:
- /* Disable access to user memory */
- csrc CSR_STATUS, t6
-- mv a0, a1
-+ sub a0, a3, a0
- ret
- ENDPROC(__clear_user)
- EXPORT_SYMBOL(__clear_user)
---
-2.40.1
-
projects / thirdparty / kernel / stable-queue.git / commitdiff
