to run kernel only on systems with h/w unaligned access support in
order to optimise for performance.
+config CPU_HAS_AMO
+ bool
+ default 64BIT
+
config CPU_HAS_FPU
bool
default y
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Atomic operations (AMO).
+ *
+ * Copyright (C) 2020-2025 Loongson Technology Corporation Limited
+ */
+
+#ifndef _ASM_ATOMIC_AMO_H
+#define _ASM_ATOMIC_AMO_H
+
+#include <linux/types.h>
+#include <asm/barrier.h>
+#include <asm/cmpxchg.h>
+
+#define ATOMIC_OP(op, I, asm_op) \
+static inline void arch_atomic_##op(int i, atomic_t *v) \
+{ \
+ __asm__ __volatile__( \
+ "am"#asm_op".w" " $zero, %1, %0 \n" \
+ : "+ZB" (v->counter) \
+ : "r" (I) \
+ : "memory"); \
+}
+
+#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
+static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
+{ \
+ int result; \
+ \
+ __asm__ __volatile__( \
+ "am"#asm_op#mb".w" " %1, %2, %0 \n" \
+ : "+ZB" (v->counter), "=&r" (result) \
+ : "r" (I) \
+ : "memory"); \
+ \
+ return result c_op I; \
+}
+
+#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \
+static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \
+{ \
+ int result; \
+ \
+ __asm__ __volatile__( \
+ "am"#asm_op#mb".w" " %1, %2, %0 \n" \
+ : "+ZB" (v->counter), "=&r" (result) \
+ : "r" (I) \
+ : "memory"); \
+ \
+ return result; \
+}
+
+#define ATOMIC_OPS(op, I, asm_op, c_op) \
+ ATOMIC_OP(op, I, asm_op) \
+ ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \
+ ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
+ ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
+ ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
+
+ATOMIC_OPS(add, i, add, +)
+ATOMIC_OPS(sub, -i, add, +)
+
+#define arch_atomic_add_return arch_atomic_add_return
+#define arch_atomic_add_return_acquire arch_atomic_add_return
+#define arch_atomic_add_return_release arch_atomic_add_return
+#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
+#define arch_atomic_sub_return arch_atomic_sub_return
+#define arch_atomic_sub_return_acquire arch_atomic_sub_return
+#define arch_atomic_sub_return_release arch_atomic_sub_return
+#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
+#define arch_atomic_fetch_add arch_atomic_fetch_add
+#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
+#define arch_atomic_fetch_add_release arch_atomic_fetch_add
+#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
+#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
+#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
+#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
+
+#undef ATOMIC_OPS
+
+#define ATOMIC_OPS(op, I, asm_op) \
+ ATOMIC_OP(op, I, asm_op) \
+ ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
+ ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
+
+ATOMIC_OPS(and, i, and)
+ATOMIC_OPS(or, i, or)
+ATOMIC_OPS(xor, i, xor)
+
+#define arch_atomic_fetch_and arch_atomic_fetch_and
+#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
+#define arch_atomic_fetch_and_release arch_atomic_fetch_and
+#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
+#define arch_atomic_fetch_or arch_atomic_fetch_or
+#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
+#define arch_atomic_fetch_or_release arch_atomic_fetch_or
+#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
+#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
+#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
+#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+
+#undef ATOMIC_OPS
+#undef ATOMIC_FETCH_OP
+#undef ATOMIC_OP_RETURN
+#undef ATOMIC_OP
+
+#ifdef CONFIG_64BIT
+
+#define ATOMIC64_OP(op, I, asm_op) \
+static inline void arch_atomic64_##op(long i, atomic64_t *v) \
+{ \
+ __asm__ __volatile__( \
+ "am"#asm_op".d " " $zero, %1, %0 \n" \
+ : "+ZB" (v->counter) \
+ : "r" (I) \
+ : "memory"); \
+}
+
+#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
+static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \
+{ \
+ long result; \
+ __asm__ __volatile__( \
+ "am"#asm_op#mb".d " " %1, %2, %0 \n" \
+ : "+ZB" (v->counter), "=&r" (result) \
+ : "r" (I) \
+ : "memory"); \
+ \
+ return result c_op I; \
+}
+
+#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \
+static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \
+{ \
+ long result; \
+ \
+ __asm__ __volatile__( \
+ "am"#asm_op#mb".d " " %1, %2, %0 \n" \
+ : "+ZB" (v->counter), "=&r" (result) \
+ : "r" (I) \
+ : "memory"); \
+ \
+ return result; \
+}
+
+#define ATOMIC64_OPS(op, I, asm_op, c_op) \
+ ATOMIC64_OP(op, I, asm_op) \
+ ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \
+ ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
+ ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
+ ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
+
+ATOMIC64_OPS(add, i, add, +)
+ATOMIC64_OPS(sub, -i, add, +)
+
+#define arch_atomic64_add_return arch_atomic64_add_return
+#define arch_atomic64_add_return_acquire arch_atomic64_add_return
+#define arch_atomic64_add_return_release arch_atomic64_add_return
+#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
+#define arch_atomic64_sub_return arch_atomic64_sub_return
+#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
+#define arch_atomic64_sub_return_release arch_atomic64_sub_return
+#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
+#define arch_atomic64_fetch_add arch_atomic64_fetch_add
+#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
+#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
+#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
+#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
+#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
+#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
+#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
+
+#undef ATOMIC64_OPS
+
+#define ATOMIC64_OPS(op, I, asm_op) \
+ ATOMIC64_OP(op, I, asm_op) \
+ ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
+ ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
+
+ATOMIC64_OPS(and, i, and)
+ATOMIC64_OPS(or, i, or)
+ATOMIC64_OPS(xor, i, xor)
+
+#define arch_atomic64_fetch_and arch_atomic64_fetch_and
+#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
+#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
+#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
+#define arch_atomic64_fetch_or arch_atomic64_fetch_or
+#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
+#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
+#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
+#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
+#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
+#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
+#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
+
+#undef ATOMIC64_OPS
+#undef ATOMIC64_FETCH_OP
+#undef ATOMIC64_OP_RETURN
+#undef ATOMIC64_OP
+
+#endif
+
+#endif /* _ASM_ATOMIC_AMO_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Atomic operations (LLSC).
+ *
+ * Copyright (C) 2024-2025 Loongson Technology Corporation Limited
+ */
+
+#ifndef _ASM_ATOMIC_LLSC_H
+#define _ASM_ATOMIC_LLSC_H
+
+#include <linux/types.h>
+#include <asm/barrier.h>
+#include <asm/cmpxchg.h>
+
+#define ATOMIC_OP(op, I, asm_op) \
+static inline void arch_atomic_##op(int i, atomic_t *v) \
+{ \
+ int temp; \
+ \
+ __asm__ __volatile__( \
+ "1: ll.w %0, %1 #atomic_" #op " \n" \
+ " " #asm_op " %0, %0, %2 \n" \
+ " sc.w %0, %1 \n" \
+ " beq %0, $r0, 1b \n" \
+ :"=&r" (temp) , "+ZC"(v->counter) \
+ :"r" (I) \
+ ); \
+}
+
+#define ATOMIC_OP_RETURN(op, I, asm_op) \
+static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
+{ \
+ int result, temp; \
+ \
+ __asm__ __volatile__( \
+ "1: ll.w %1, %2 # atomic_" #op "_return \n" \
+ " " #asm_op " %0, %1, %3 \n" \
+ " sc.w %0, %2 \n" \
+ " beq %0, $r0 ,1b \n" \
+ " " #asm_op " %0, %1, %3 \n" \
+ : "=&r" (result), "=&r" (temp), "+ZC"(v->counter) \
+ : "r" (I)); \
+ \
+ return result; \
+}
+
+#define ATOMIC_FETCH_OP(op, I, asm_op) \
+static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
+{ \
+ int result, temp; \
+ \
+ __asm__ __volatile__( \
+ "1: ll.w %1, %2 # atomic_fetch_" #op " \n" \
+ " " #asm_op " %0, %1, %3 \n" \
+ " sc.w %0, %2 \n" \
+ " beq %0, $r0 ,1b \n" \
+ " add.w %0, %1 ,$r0 \n" \
+ : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) \
+ : "r" (I)); \
+ \
+ return result; \
+}
+
+#define ATOMIC_OPS(op,I ,asm_op, c_op) \
+ ATOMIC_OP(op, I, asm_op) \
+ ATOMIC_OP_RETURN(op, I , asm_op) \
+ ATOMIC_FETCH_OP(op, I, asm_op)
+
+ATOMIC_OPS(add, i , add.w ,+=)
+ATOMIC_OPS(sub, -i , add.w ,+=)
+
+#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
+#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
+#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
+#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
+
+#undef ATOMIC_OPS
+
+#define ATOMIC_OPS(op, I, asm_op) \
+ ATOMIC_OP(op, I, asm_op) \
+ ATOMIC_FETCH_OP(op, I, asm_op)
+
+ATOMIC_OPS(and, i, and)
+ATOMIC_OPS(or, i, or)
+ATOMIC_OPS(xor, i, xor)
+
+#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
+#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
+#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+
+#undef ATOMIC_OPS
+#undef ATOMIC_FETCH_OP
+#undef ATOMIC_OP_RETURN
+#undef ATOMIC_OP
+
+#ifdef CONFIG_64BIT
+#error "64-bit LLSC atomic operations are not supported"
+#endif
+
+#endif /* _ASM_ATOMIC_LLSC_H */
#include <asm/barrier.h>
#include <asm/cmpxchg.h>
+#ifdef CONFIG_CPU_HAS_AMO
+#include <asm/atomic-amo.h>
+#else
+#include <asm/atomic-llsc.h>
+#endif
+
+#ifdef CONFIG_GENERIC_ATOMIC64
+#include <asm-generic/atomic64.h>
+#endif
+
#if __SIZEOF_LONG__ == 4
#define __LL "ll.w "
#define __SC "sc.w "
#define arch_atomic_read(v) READ_ONCE((v)->counter)
#define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))
-#define ATOMIC_OP(op, I, asm_op) \
-static inline void arch_atomic_##op(int i, atomic_t *v) \
-{ \
- __asm__ __volatile__( \
- "am"#asm_op".w" " $zero, %1, %0 \n" \
- : "+ZB" (v->counter) \
- : "r" (I) \
- : "memory"); \
-}
-
-#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
-static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
-{ \
- int result; \
- \
- __asm__ __volatile__( \
- "am"#asm_op#mb".w" " %1, %2, %0 \n" \
- : "+ZB" (v->counter), "=&r" (result) \
- : "r" (I) \
- : "memory"); \
- \
- return result c_op I; \
-}
-
-#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \
-static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \
-{ \
- int result; \
- \
- __asm__ __volatile__( \
- "am"#asm_op#mb".w" " %1, %2, %0 \n" \
- : "+ZB" (v->counter), "=&r" (result) \
- : "r" (I) \
- : "memory"); \
- \
- return result; \
-}
-
-#define ATOMIC_OPS(op, I, asm_op, c_op) \
- ATOMIC_OP(op, I, asm_op) \
- ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \
- ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
- ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
- ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
-
-ATOMIC_OPS(add, i, add, +)
-ATOMIC_OPS(sub, -i, add, +)
-
-#define arch_atomic_add_return arch_atomic_add_return
-#define arch_atomic_add_return_acquire arch_atomic_add_return
-#define arch_atomic_add_return_release arch_atomic_add_return
-#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
-#define arch_atomic_sub_return arch_atomic_sub_return
-#define arch_atomic_sub_return_acquire arch_atomic_sub_return
-#define arch_atomic_sub_return_release arch_atomic_sub_return
-#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
-#define arch_atomic_fetch_add arch_atomic_fetch_add
-#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
-#define arch_atomic_fetch_add_release arch_atomic_fetch_add
-#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
-#define arch_atomic_fetch_sub arch_atomic_fetch_sub
-#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
-#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
-#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
-
-#undef ATOMIC_OPS
-
-#define ATOMIC_OPS(op, I, asm_op) \
- ATOMIC_OP(op, I, asm_op) \
- ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
- ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
-
-ATOMIC_OPS(and, i, and)
-ATOMIC_OPS(or, i, or)
-ATOMIC_OPS(xor, i, xor)
-
-#define arch_atomic_fetch_and arch_atomic_fetch_and
-#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
-#define arch_atomic_fetch_and_release arch_atomic_fetch_and
-#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
-#define arch_atomic_fetch_or arch_atomic_fetch_or
-#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
-#define arch_atomic_fetch_or_release arch_atomic_fetch_or
-#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
-#define arch_atomic_fetch_xor arch_atomic_fetch_xor
-#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
-#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
-#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
-
-#undef ATOMIC_OPS
-#undef ATOMIC_FETCH_OP
-#undef ATOMIC_OP_RETURN
-#undef ATOMIC_OP
-
static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
{
int prev, rc;
#define arch_atomic64_read(v) READ_ONCE((v)->counter)
#define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
-#define ATOMIC64_OP(op, I, asm_op) \
-static inline void arch_atomic64_##op(long i, atomic64_t *v) \
-{ \
- __asm__ __volatile__( \
- "am"#asm_op".d " " $zero, %1, %0 \n" \
- : "+ZB" (v->counter) \
- : "r" (I) \
- : "memory"); \
-}
-
-#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
-static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \
-{ \
- long result; \
- __asm__ __volatile__( \
- "am"#asm_op#mb".d " " %1, %2, %0 \n" \
- : "+ZB" (v->counter), "=&r" (result) \
- : "r" (I) \
- : "memory"); \
- \
- return result c_op I; \
-}
-
-#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \
-static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \
-{ \
- long result; \
- \
- __asm__ __volatile__( \
- "am"#asm_op#mb".d " " %1, %2, %0 \n" \
- : "+ZB" (v->counter), "=&r" (result) \
- : "r" (I) \
- : "memory"); \
- \
- return result; \
-}
-
-#define ATOMIC64_OPS(op, I, asm_op, c_op) \
- ATOMIC64_OP(op, I, asm_op) \
- ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \
- ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
- ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
- ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
-
-ATOMIC64_OPS(add, i, add, +)
-ATOMIC64_OPS(sub, -i, add, +)
-
-#define arch_atomic64_add_return arch_atomic64_add_return
-#define arch_atomic64_add_return_acquire arch_atomic64_add_return
-#define arch_atomic64_add_return_release arch_atomic64_add_return
-#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
-#define arch_atomic64_sub_return arch_atomic64_sub_return
-#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
-#define arch_atomic64_sub_return_release arch_atomic64_sub_return
-#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
-#define arch_atomic64_fetch_add arch_atomic64_fetch_add
-#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
-#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
-#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
-#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
-#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
-#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
-#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
-
-#undef ATOMIC64_OPS
-
-#define ATOMIC64_OPS(op, I, asm_op) \
- ATOMIC64_OP(op, I, asm_op) \
- ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
- ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
-
-ATOMIC64_OPS(and, i, and)
-ATOMIC64_OPS(or, i, or)
-ATOMIC64_OPS(xor, i, xor)
-
-#define arch_atomic64_fetch_and arch_atomic64_fetch_and
-#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
-#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
-#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
-#define arch_atomic64_fetch_or arch_atomic64_fetch_or
-#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
-#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
-#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
-#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
-#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
-#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
-#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
-
-#undef ATOMIC64_OPS
-#undef ATOMIC64_FETCH_OP
-#undef ATOMIC64_OP_RETURN
-#undef ATOMIC64_OP
-
static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
{
long prev, rc;
#include <linux/build_bug.h>
#include <asm/barrier.h>
-#define __xchg_asm(amswap_db, m, val) \
+#define __xchg_amo_asm(amswap_db, m, val) \
({ \
- __typeof(val) __ret; \
+ __typeof(val) __ret; \
\
- __asm__ __volatile__ ( \
- " "amswap_db" %1, %z2, %0 \n" \
- : "+ZB" (*m), "=&r" (__ret) \
- : "Jr" (val) \
- : "memory"); \
+ __asm__ __volatile__ ( \
+ " "amswap_db" %1, %z2, %0 \n" \
+ : "+ZB" (*m), "=&r" (__ret) \
+ : "Jr" (val) \
+ : "memory"); \
\
- __ret; \
+ __ret; \
+})
+
+#define __xchg_llsc_asm(ld, st, m, val) \
+({ \
+ __typeof(val) __ret, __tmp; \
+ \
+ asm volatile ( \
+ "1: ll.w %0, %3 \n" \
+ " move %1, %z4 \n" \
+ " sc.w %1, %2 \n" \
+ " beqz %1, 1b \n" \
+ : "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m) \
+ : "ZC" (*m), "Jr" (val) \
+ : "memory"); \
+ \
+ __ret; \
})
static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,
switch (size) {
case 1:
case 2:
- return __xchg_small(ptr, x, size);
+ return __xchg_small((volatile void *)ptr, x, size);
case 4:
- return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
+#ifdef CONFIG_CPU_HAS_AMO
+ return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
+#else
+ return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x);
+#endif /* CONFIG_CPU_HAS_AMO */
+#ifdef CONFIG_64BIT
case 8:
- return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
+#ifdef CONFIG_CPU_HAS_AMO
+ return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
+#else
+ return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x);
+#endif /* CONFIG_CPU_HAS_AMO */
+#endif /* CONFIG_64BIT */
default:
BUILD_BUG();
#include <linux/percpu.h>
#include <linux/bitops.h>
#include <linux/atomic.h>
+#include <asm/asm.h>
#include <asm/cmpxchg.h>
typedef struct {
/*
* Same as above, but return the result value
*/
+#ifdef CONFIG_CPU_HAS_AMO
static inline long local_add_return(long i, local_t *l)
{
unsigned long result;
return result;
}
+#else
+static inline long local_add_return(long i, local_t *l)
+{
+ unsigned long result, temp;
+
+ __asm__ __volatile__(
+ "1:" __LL "%1, %2 # local_add_return \n"
+ __stringify(LONG_ADD) " %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beq %0, $r0, 1b \n"
+ __stringify(LONG_ADD) " %0, %1, %3 \n"
+ : "=&r" (result), "=&r" (temp), "=ZC" (l->a.counter)
+ : "r" (i), "ZC" (l->a.counter)
+ : "memory");
+
+ return result;
+}
+
+static inline long local_sub_return(long i, local_t *l)
+{
+ unsigned long result, temp;
+
+ __asm__ __volatile__(
+ "1:" __LL "%1, %2 # local_sub_return \n"
+ __stringify(LONG_SUB) " %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beq %0, $r0, 1b \n"
+ __stringify(LONG_SUB) " %0, %1, %3 \n"
+ : "=&r" (result), "=&r" (temp), "=ZC" (l->a.counter)
+ : "r" (i), "ZC" (l->a.counter)
+ : "memory");
+
+ return result;
+}
+#endif
static inline long local_cmpxchg(local_t *l, long old, long new)
{
__my_cpu_offset; \
})
+#ifdef CONFIG_CPU_HAS_AMO
+
#define PERCPU_OP(op, asm_op, c_op) \
static __always_inline unsigned long __percpu_##op(void *ptr, \
unsigned long val, int size) \
PERCPU_OP(or, or, |)
#undef PERCPU_OP
-static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
-{
- switch (size) {
- case 1:
- case 2:
- return __xchg_small((volatile void *)ptr, val, size);
-
- case 4:
- return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
-
- case 8:
- return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
+#endif
- default:
- BUILD_BUG();
- }
-
- return 0;
-}
+#ifdef CONFIG_64BIT
#define __pcpu_op_1(op) op ".b "
#define __pcpu_op_2(op) op ".h "
: "memory"); \
} while (0)
+#endif
+
+#define __percpu_xchg __arch_xchg
+
/* this_cpu_cmpxchg */
#define _protect_cmpxchg_local(pcp, o, n) \
({ \
__retval; \
})
+#ifdef CONFIG_CPU_HAS_AMO
+
#define _percpu_add(pcp, val) \
_pcp_protect(__percpu_add, pcp, val)
#define _percpu_or(pcp, val) \
_pcp_protect(__percpu_or, pcp, val)
-#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
- _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
-
#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+#endif
+
+#ifdef CONFIG_64BIT
+
#define this_cpu_read_1(pcp) _percpu_read(1, pcp)
#define this_cpu_read_2(pcp) _percpu_read(2, pcp)
#define this_cpu_read_4(pcp) _percpu_read(4, pcp)
#define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val)
#define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val)
+#endif
+
+#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
+ _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
+
#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
projects / thirdparty / linux.git / commitdiff
