locking/atomic/x86: Introduce the read64_nonatomic macro to x86_32 with cx8

As described in commit:

  e73c4e34a0e9 ("locking/atomic/x86: Introduce arch_atomic64_read_nonatomic() to x86_32")

the value preload before the CMPXCHG loop does not need to be atomic.

Introduce the read64_nonatomic assembly macro to load the value from a
atomic_t location in a faster non-atomic way and use it in
atomic64_cx8_32.S.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240605181424.3228-1-ubizjak@gmail.com

diff --git a/arch/x86/lib/atomic64_cx8_32.S b/arch/x86/lib/atomic64_cx8_32.S
index 90afb488b396a0eedf974127e81b441185ccadd2..b2eff07d65e47e2ad7f987b1ed54587bf603bee3 100644 (file)
--- a/arch/x86/lib/atomic64_cx8_32.S
+++ b/arch/x86/lib/atomic64_cx8_32.S
@@ -16,6 +16,11 @@
        cmpxchg8b (\reg)
 .endm
 
+.macro read64_nonatomic reg
+       movl (\reg), %eax
+       movl 4(\reg), %edx
+.endm
+
 SYM_FUNC_START(atomic64_read_cx8)
        read64 %ecx
        RET
@@ -51,7 +56,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8)
        movl %edx, %edi
        movl %ecx, %ebp
 
-       read64 %ecx
+       read64_nonatomic %ecx
 1:
        movl %eax, %ebx
        movl %edx, %ecx
@@ -79,7 +84,7 @@ addsub_return sub sub sbb
 SYM_FUNC_START(atomic64_\func\()_return_cx8)
        pushl %ebx
 
-       read64 %esi
+       read64_nonatomic %esi
 1:
        movl %eax, %ebx
        movl %edx, %ecx