--- /dev/null
+From 7df8ac8777bc608ca41b7f9defd14bf710bdca7b Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Wed, 10 Nov 2021 11:01:03 +0100
+Subject: bitfield.h: Fix "type of reg too small for mask" test
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+[ Upstream commit bff8c3848e071d387d8b0784dc91fa49cd563774 ]
+
+The test: 'mask > (typeof(_reg))~0ull' only works correctly when both
+sides are unsigned, consider:
+
+ - 0xff000000 vs (int)~0ull
+ - 0x000000ff vs (int)~0ull
+
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com>
+Link: https://lore.kernel.org/r/20211110101324.950210584@infradead.org
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/bitfield.h | 19 ++++++++++++++++++-
+ 1 file changed, 18 insertions(+), 1 deletion(-)
+
+diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h
+index 4c0224ff0a14..4f1c0f8e1bb0 100644
+--- a/include/linux/bitfield.h
++++ b/include/linux/bitfield.h
+@@ -41,6 +41,22 @@
+
+ #define __bf_shf(x) (__builtin_ffsll(x) - 1)
+
++#define __scalar_type_to_unsigned_cases(type) \
++ unsigned type: (unsigned type)0, \
++ signed type: (unsigned type)0
++
++#define __unsigned_scalar_typeof(x) typeof( \
++ _Generic((x), \
++ char: (unsigned char)0, \
++ __scalar_type_to_unsigned_cases(char), \
++ __scalar_type_to_unsigned_cases(short), \
++ __scalar_type_to_unsigned_cases(int), \
++ __scalar_type_to_unsigned_cases(long), \
++ __scalar_type_to_unsigned_cases(long long), \
++ default: (x)))
++
++#define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x))
++
+ #define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \
+ ({ \
+ BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \
+@@ -49,7 +65,8 @@
+ BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \
+ ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \
+ _pfx "value too large for the field"); \
+- BUILD_BUG_ON_MSG((_mask) > (typeof(_reg))~0ull, \
++ BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \
++ __bf_cast_unsigned(_reg, ~0ull), \
+ _pfx "type of reg too small for mask"); \
+ __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \
+ (1ULL << __bf_shf(_mask))); \
+--
+2.35.1
+
--- /dev/null
+From 2445f6197707f527534b5277ebc0689c6211174f Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Wed, 6 Apr 2022 13:34:16 -0400
+Subject: dlm: fix pending remove if msg allocation fails
+
+From: Alexander Aring <aahringo@redhat.com>
+
+[ Upstream commit ba58995909b5098ca4003af65b0ccd5a8d13dd25 ]
+
+This patch unsets ls_remove_len and ls_remove_name if a message
+allocation of a remove messages fails. In this case we never send a
+remove message out but set the per ls ls_remove_len ls_remove_name
+variable for a pending remove. Unset those variable should indicate
+possible waiters in wait_pending_remove() that no pending remove is
+going on at this moment.
+
+Cc: stable@vger.kernel.org
+Signed-off-by: Alexander Aring <aahringo@redhat.com>
+Signed-off-by: David Teigland <teigland@redhat.com>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ fs/dlm/lock.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c
+index 4ae8becdb51d..9165bf56c6e8 100644
+--- a/fs/dlm/lock.c
++++ b/fs/dlm/lock.c
+@@ -4067,13 +4067,14 @@ static void send_repeat_remove(struct dlm_ls *ls, char *ms_name, int len)
+ rv = _create_message(ls, sizeof(struct dlm_message) + len,
+ dir_nodeid, DLM_MSG_REMOVE, &ms, &mh);
+ if (rv)
+- return;
++ goto out;
+
+ memcpy(ms->m_extra, name, len);
+ ms->m_hash = hash;
+
+ send_message(mh, ms);
+
++out:
+ spin_lock(&ls->ls_remove_spin);
+ ls->ls_remove_len = 0;
+ memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN);
+--
+2.35.1
+
--- /dev/null
+From 96bd7f547174e7069b0cd92f4d30e363b5f06e23 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 7 Apr 2022 10:16:19 +0800
+Subject: ima: remove the IMA_TEMPLATE Kconfig option
+
+From: GUO Zihua <guozihua@huawei.com>
+
+[ Upstream commit 891163adf180bc369b2f11c9dfce6d2758d2a5bd ]
+
+The original 'ima' measurement list template contains a hash, defined
+as 20 bytes, and a null terminated pathname, limited to 255
+characters. Other measurement list templates permit both larger hashes
+and longer pathnames. When the "ima" template is configured as the
+default, a new measurement list template (ima_template=) must be
+specified before specifying a larger hash algorithm (ima_hash=) on the
+boot command line.
+
+To avoid this boot command line ordering issue, remove the legacy "ima"
+template configuration option, allowing it to still be specified on the
+boot command line.
+
+The root cause of this issue is that during the processing of ima_hash,
+we would try to check whether the hash algorithm is compatible with the
+template. If the template is not set at the moment we do the check, we
+check the algorithm against the configured default template. If the
+default template is "ima", then we reject any hash algorithm other than
+sha1 and md5.
+
+For example, if the compiled default template is "ima", and the default
+algorithm is sha1 (which is the current default). In the cmdline, we put
+in "ima_hash=sha256 ima_template=ima-ng". The expected behavior would be
+that ima starts with ima-ng as the template and sha256 as the hash
+algorithm. However, during the processing of "ima_hash=",
+"ima_template=" has not been processed yet, and hash_setup would check
+the configured hash algorithm against the compiled default: ima, and
+reject sha256. So at the end, the hash algorithm that is actually used
+will be sha1.
+
+With template "ima" removed from the configured default, we ensure that
+the default tempalte would at least be "ima-ng" which allows for
+basically any hash algorithm.
+
+This change would not break the algorithm compatibility checks for IMA.
+
+Fixes: 4286587dccd43 ("ima: add Kconfig default measurement list template")
+Signed-off-by: GUO Zihua <guozihua@huawei.com>
+Cc: <Stable@vger.kernel.org>
+Signed-off-by: Mimi Zohar <zohar@linux.ibm.com>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ security/integrity/ima/Kconfig | 12 +++++-------
+ 1 file changed, 5 insertions(+), 7 deletions(-)
+
+diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig
+index 748f3ee27b23..44b3315f3235 100644
+--- a/security/integrity/ima/Kconfig
++++ b/security/integrity/ima/Kconfig
+@@ -69,10 +69,9 @@ choice
+ hash, defined as 20 bytes, and a null terminated pathname,
+ limited to 255 characters. The 'ima-ng' measurement list
+ template permits both larger hash digests and longer
+- pathnames.
++ pathnames. The configured default template can be replaced
++ by specifying "ima_template=" on the boot command line.
+
+- config IMA_TEMPLATE
+- bool "ima"
+ config IMA_NG_TEMPLATE
+ bool "ima-ng (default)"
+ config IMA_SIG_TEMPLATE
+@@ -82,7 +81,6 @@ endchoice
+ config IMA_DEFAULT_TEMPLATE
+ string
+ depends on IMA
+- default "ima" if IMA_TEMPLATE
+ default "ima-ng" if IMA_NG_TEMPLATE
+ default "ima-sig" if IMA_SIG_TEMPLATE
+
+@@ -102,15 +100,15 @@ choice
+
+ config IMA_DEFAULT_HASH_SHA256
+ bool "SHA256"
+- depends on CRYPTO_SHA256=y && !IMA_TEMPLATE
++ depends on CRYPTO_SHA256=y
+
+ config IMA_DEFAULT_HASH_SHA512
+ bool "SHA512"
+- depends on CRYPTO_SHA512=y && !IMA_TEMPLATE
++ depends on CRYPTO_SHA512=y
+
+ config IMA_DEFAULT_HASH_WP512
+ bool "WP512"
+- depends on CRYPTO_WP512=y && !IMA_TEMPLATE
++ depends on CRYPTO_WP512=y
+ endchoice
+
+ config IMA_DEFAULT_HASH
+--
+2.35.1
+
--- /dev/null
+From 4afe87339436d8e4a820c4ad24a9e3880e45a6c4 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:59:00 +0000
+Subject: locking/refcount: Consolidate implementations of refcount_t
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit fb041bb7c0a918b95c6889fc965cdc4a75b4c0ca ]
+
+The generic implementation of refcount_t should be good enough for
+everybody, so remove ARCH_HAS_REFCOUNT and REFCOUNT_FULL entirely,
+leaving the generic implementation enabled unconditionally.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Acked-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-9-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ arch/Kconfig | 21 ----
+ arch/arm/Kconfig | 1 -
+ arch/arm64/Kconfig | 1 -
+ arch/s390/configs/debug_defconfig | 1 -
+ arch/x86/Kconfig | 1 -
+ arch/x86/include/asm/asm.h | 6 --
+ arch/x86/include/asm/refcount.h | 126 -----------------------
+ arch/x86/mm/extable.c | 49 ---------
+ drivers/gpu/drm/i915/Kconfig.debug | 1 -
+ include/linux/refcount.h | 158 +++++++++++------------------
+ lib/refcount.c | 2 +-
+ 11 files changed, 59 insertions(+), 308 deletions(-)
+ delete mode 100644 arch/x86/include/asm/refcount.h
+
+diff --git a/arch/Kconfig b/arch/Kconfig
+index a8df66e64544..2219a07dca1e 100644
+--- a/arch/Kconfig
++++ b/arch/Kconfig
+@@ -915,27 +915,6 @@ config STRICT_MODULE_RWX
+ config ARCH_HAS_PHYS_TO_DMA
+ bool
+
+-config ARCH_HAS_REFCOUNT
+- bool
+- help
+- An architecture selects this when it has implemented refcount_t
+- using open coded assembly primitives that provide an optimized
+- refcount_t implementation, possibly at the expense of some full
+- refcount state checks of CONFIG_REFCOUNT_FULL=y.
+-
+- The refcount overflow check behavior, however, must be retained.
+- Catching overflows is the primary security concern for protecting
+- against bugs in reference counts.
+-
+-config REFCOUNT_FULL
+- bool "Perform full reference count validation at the expense of speed"
+- help
+- Enabling this switches the refcounting infrastructure from a fast
+- unchecked atomic_t implementation to a fully state checked
+- implementation, which can be (slightly) slower but provides protections
+- against various use-after-free conditions that can be used in
+- security flaw exploits.
+-
+ config HAVE_ARCH_COMPILER_H
+ bool
+ help
+diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
+index a1622b9290fd..a4364cce85f8 100644
+--- a/arch/arm/Kconfig
++++ b/arch/arm/Kconfig
+@@ -119,7 +119,6 @@ config ARM
+ select OLD_SIGSUSPEND3
+ select PCI_SYSCALL if PCI
+ select PERF_USE_VMALLOC
+- select REFCOUNT_FULL
+ select RTC_LIB
+ select SYS_SUPPORTS_APM_EMULATION
+ # Above selects are sorted alphabetically; please add new ones
+diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
+index a1a828ca188c..6b73143f0cf8 100644
+--- a/arch/arm64/Kconfig
++++ b/arch/arm64/Kconfig
+@@ -181,7 +181,6 @@ config ARM64
+ select PCI_SYSCALL if PCI
+ select POWER_RESET
+ select POWER_SUPPLY
+- select REFCOUNT_FULL
+ select SPARSE_IRQ
+ select SWIOTLB
+ select SYSCTL_EXCEPTION_TRACE
+diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
+index 38d64030aacf..2e60c80395ab 100644
+--- a/arch/s390/configs/debug_defconfig
++++ b/arch/s390/configs/debug_defconfig
+@@ -62,7 +62,6 @@ CONFIG_OPROFILE=m
+ CONFIG_KPROBES=y
+ CONFIG_JUMP_LABEL=y
+ CONFIG_STATIC_KEYS_SELFTEST=y
+-CONFIG_REFCOUNT_FULL=y
+ CONFIG_LOCK_EVENT_COUNTS=y
+ CONFIG_MODULES=y
+ CONFIG_MODULE_FORCE_LOAD=y
+diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
+index c6c71592f6e4..6002252692af 100644
+--- a/arch/x86/Kconfig
++++ b/arch/x86/Kconfig
+@@ -73,7 +73,6 @@ config X86
+ select ARCH_HAS_PMEM_API if X86_64
+ select ARCH_HAS_PTE_DEVMAP if X86_64
+ select ARCH_HAS_PTE_SPECIAL
+- select ARCH_HAS_REFCOUNT
+ select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64
+ select ARCH_HAS_UACCESS_MCSAFE if X86_64 && X86_MCE
+ select ARCH_HAS_SET_MEMORY
+diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
+index 1b563f9167ea..cd339b88d5d4 100644
+--- a/arch/x86/include/asm/asm.h
++++ b/arch/x86/include/asm/asm.h
+@@ -141,9 +141,6 @@
+ # define _ASM_EXTABLE_EX(from, to) \
+ _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext)
+
+-# define _ASM_EXTABLE_REFCOUNT(from, to) \
+- _ASM_EXTABLE_HANDLE(from, to, ex_handler_refcount)
+-
+ # define _ASM_NOKPROBE(entry) \
+ .pushsection "_kprobe_blacklist","aw" ; \
+ _ASM_ALIGN ; \
+@@ -172,9 +169,6 @@
+ # define _ASM_EXTABLE_EX(from, to) \
+ _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext)
+
+-# define _ASM_EXTABLE_REFCOUNT(from, to) \
+- _ASM_EXTABLE_HANDLE(from, to, ex_handler_refcount)
+-
+ /* For C file, we already have NOKPROBE_SYMBOL macro */
+ #endif
+
+diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h
+deleted file mode 100644
+index 232f856e0db0..000000000000
+--- a/arch/x86/include/asm/refcount.h
++++ /dev/null
+@@ -1,126 +0,0 @@
+-#ifndef __ASM_X86_REFCOUNT_H
+-#define __ASM_X86_REFCOUNT_H
+-/*
+- * x86-specific implementation of refcount_t. Based on PAX_REFCOUNT from
+- * PaX/grsecurity.
+- */
+-#include <linux/refcount.h>
+-#include <asm/bug.h>
+-
+-/*
+- * This is the first portion of the refcount error handling, which lives in
+- * .text.unlikely, and is jumped to from the CPU flag check (in the
+- * following macros). This saves the refcount value location into CX for
+- * the exception handler to use (in mm/extable.c), and then triggers the
+- * central refcount exception. The fixup address for the exception points
+- * back to the regular execution flow in .text.
+- */
+-#define _REFCOUNT_EXCEPTION \
+- ".pushsection .text..refcount\n" \
+- "111:\tlea %[var], %%" _ASM_CX "\n" \
+- "112:\t" ASM_UD2 "\n" \
+- ASM_UNREACHABLE \
+- ".popsection\n" \
+- "113:\n" \
+- _ASM_EXTABLE_REFCOUNT(112b, 113b)
+-
+-/* Trigger refcount exception if refcount result is negative. */
+-#define REFCOUNT_CHECK_LT_ZERO \
+- "js 111f\n\t" \
+- _REFCOUNT_EXCEPTION
+-
+-/* Trigger refcount exception if refcount result is zero or negative. */
+-#define REFCOUNT_CHECK_LE_ZERO \
+- "jz 111f\n\t" \
+- REFCOUNT_CHECK_LT_ZERO
+-
+-/* Trigger refcount exception unconditionally. */
+-#define REFCOUNT_ERROR \
+- "jmp 111f\n\t" \
+- _REFCOUNT_EXCEPTION
+-
+-static __always_inline void refcount_add(unsigned int i, refcount_t *r)
+-{
+- asm volatile(LOCK_PREFIX "addl %1,%0\n\t"
+- REFCOUNT_CHECK_LT_ZERO
+- : [var] "+m" (r->refs.counter)
+- : "ir" (i)
+- : "cc", "cx");
+-}
+-
+-static __always_inline void refcount_inc(refcount_t *r)
+-{
+- asm volatile(LOCK_PREFIX "incl %0\n\t"
+- REFCOUNT_CHECK_LT_ZERO
+- : [var] "+m" (r->refs.counter)
+- : : "cc", "cx");
+-}
+-
+-static __always_inline void refcount_dec(refcount_t *r)
+-{
+- asm volatile(LOCK_PREFIX "decl %0\n\t"
+- REFCOUNT_CHECK_LE_ZERO
+- : [var] "+m" (r->refs.counter)
+- : : "cc", "cx");
+-}
+-
+-static __always_inline __must_check
+-bool refcount_sub_and_test(unsigned int i, refcount_t *r)
+-{
+- bool ret = GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl",
+- REFCOUNT_CHECK_LT_ZERO,
+- r->refs.counter, e, "er", i, "cx");
+-
+- if (ret) {
+- smp_acquire__after_ctrl_dep();
+- return true;
+- }
+-
+- return false;
+-}
+-
+-static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r)
+-{
+- bool ret = GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl",
+- REFCOUNT_CHECK_LT_ZERO,
+- r->refs.counter, e, "cx");
+-
+- if (ret) {
+- smp_acquire__after_ctrl_dep();
+- return true;
+- }
+-
+- return false;
+-}
+-
+-static __always_inline __must_check
+-bool refcount_add_not_zero(unsigned int i, refcount_t *r)
+-{
+- int c, result;
+-
+- c = atomic_read(&(r->refs));
+- do {
+- if (unlikely(c == 0))
+- return false;
+-
+- result = c + i;
+-
+- /* Did we try to increment from/to an undesirable state? */
+- if (unlikely(c < 0 || c == INT_MAX || result < c)) {
+- asm volatile(REFCOUNT_ERROR
+- : : [var] "m" (r->refs.counter)
+- : "cc", "cx");
+- break;
+- }
+-
+- } while (!atomic_try_cmpxchg(&(r->refs), &c, result));
+-
+- return c != 0;
+-}
+-
+-static __always_inline __must_check bool refcount_inc_not_zero(refcount_t *r)
+-{
+- return refcount_add_not_zero(1, r);
+-}
+-
+-#endif
+diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
+index 4d75bc656f97..30bb0bd3b1b8 100644
+--- a/arch/x86/mm/extable.c
++++ b/arch/x86/mm/extable.c
+@@ -44,55 +44,6 @@ __visible bool ex_handler_fault(const struct exception_table_entry *fixup,
+ }
+ EXPORT_SYMBOL_GPL(ex_handler_fault);
+
+-/*
+- * Handler for UD0 exception following a failed test against the
+- * result of a refcount inc/dec/add/sub.
+- */
+-__visible bool ex_handler_refcount(const struct exception_table_entry *fixup,
+- struct pt_regs *regs, int trapnr,
+- unsigned long error_code,
+- unsigned long fault_addr)
+-{
+- /* First unconditionally saturate the refcount. */
+- *(int *)regs->cx = INT_MIN / 2;
+-
+- /*
+- * Strictly speaking, this reports the fixup destination, not
+- * the fault location, and not the actually overflowing
+- * instruction, which is the instruction before the "js", but
+- * since that instruction could be a variety of lengths, just
+- * report the location after the overflow, which should be close
+- * enough for finding the overflow, as it's at least back in
+- * the function, having returned from .text.unlikely.
+- */
+- regs->ip = ex_fixup_addr(fixup);
+-
+- /*
+- * This function has been called because either a negative refcount
+- * value was seen by any of the refcount functions, or a zero
+- * refcount value was seen by refcount_dec().
+- *
+- * If we crossed from INT_MAX to INT_MIN, OF (Overflow Flag: result
+- * wrapped around) will be set. Additionally, seeing the refcount
+- * reach 0 will set ZF (Zero Flag: result was zero). In each of
+- * these cases we want a report, since it's a boundary condition.
+- * The SF case is not reported since it indicates post-boundary
+- * manipulations below zero or above INT_MAX. And if none of the
+- * flags are set, something has gone very wrong, so report it.
+- */
+- if (regs->flags & (X86_EFLAGS_OF | X86_EFLAGS_ZF)) {
+- bool zero = regs->flags & X86_EFLAGS_ZF;
+-
+- refcount_error_report(regs, zero ? "hit zero" : "overflow");
+- } else if ((regs->flags & X86_EFLAGS_SF) == 0) {
+- /* Report if none of OF, ZF, nor SF are set. */
+- refcount_error_report(regs, "unexpected saturation");
+- }
+-
+- return true;
+-}
+-EXPORT_SYMBOL(ex_handler_refcount);
+-
+ /*
+ * Handler for when we fail to restore a task's FPU state. We should never get
+ * here because the FPU state of a task using the FPU (task->thread.fpu.state)
+diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
+index 41c8e39a73ba..e4f03fcb125e 100644
+--- a/drivers/gpu/drm/i915/Kconfig.debug
++++ b/drivers/gpu/drm/i915/Kconfig.debug
+@@ -21,7 +21,6 @@ config DRM_I915_DEBUG
+ depends on DRM_I915
+ select DEBUG_FS
+ select PREEMPT_COUNT
+- select REFCOUNT_FULL
+ select I2C_CHARDEV
+ select STACKDEPOT
+ select DRM_DP_AUX_CHARDEV
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index 757d4630115c..0ac50cf62d06 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -1,64 +1,4 @@
+ /* SPDX-License-Identifier: GPL-2.0 */
+-#ifndef _LINUX_REFCOUNT_H
+-#define _LINUX_REFCOUNT_H
+-
+-#include <linux/atomic.h>
+-#include <linux/compiler.h>
+-#include <linux/limits.h>
+-#include <linux/spinlock_types.h>
+-
+-struct mutex;
+-
+-/**
+- * struct refcount_t - variant of atomic_t specialized for reference counts
+- * @refs: atomic_t counter field
+- *
+- * The counter saturates at REFCOUNT_SATURATED and will not move once
+- * there. This avoids wrapping the counter and causing 'spurious'
+- * use-after-free bugs.
+- */
+-typedef struct refcount_struct {
+- atomic_t refs;
+-} refcount_t;
+-
+-#define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), }
+-#define REFCOUNT_MAX INT_MAX
+-#define REFCOUNT_SATURATED (INT_MIN / 2)
+-
+-enum refcount_saturation_type {
+- REFCOUNT_ADD_NOT_ZERO_OVF,
+- REFCOUNT_ADD_OVF,
+- REFCOUNT_ADD_UAF,
+- REFCOUNT_SUB_UAF,
+- REFCOUNT_DEC_LEAK,
+-};
+-
+-void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t);
+-
+-/**
+- * refcount_set - set a refcount's value
+- * @r: the refcount
+- * @n: value to which the refcount will be set
+- */
+-static inline void refcount_set(refcount_t *r, int n)
+-{
+- atomic_set(&r->refs, n);
+-}
+-
+-/**
+- * refcount_read - get a refcount's value
+- * @r: the refcount
+- *
+- * Return: the refcount's value
+- */
+-static inline unsigned int refcount_read(const refcount_t *r)
+-{
+- return atomic_read(&r->refs);
+-}
+-
+-#ifdef CONFIG_REFCOUNT_FULL
+-#include <linux/bug.h>
+-
+ /*
+ * Variant of atomic_t specialized for reference counts.
+ *
+@@ -136,6 +76,64 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ *
+ */
+
++#ifndef _LINUX_REFCOUNT_H
++#define _LINUX_REFCOUNT_H
++
++#include <linux/atomic.h>
++#include <linux/bug.h>
++#include <linux/compiler.h>
++#include <linux/limits.h>
++#include <linux/spinlock_types.h>
++
++struct mutex;
++
++/**
++ * struct refcount_t - variant of atomic_t specialized for reference counts
++ * @refs: atomic_t counter field
++ *
++ * The counter saturates at REFCOUNT_SATURATED and will not move once
++ * there. This avoids wrapping the counter and causing 'spurious'
++ * use-after-free bugs.
++ */
++typedef struct refcount_struct {
++ atomic_t refs;
++} refcount_t;
++
++#define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), }
++#define REFCOUNT_MAX INT_MAX
++#define REFCOUNT_SATURATED (INT_MIN / 2)
++
++enum refcount_saturation_type {
++ REFCOUNT_ADD_NOT_ZERO_OVF,
++ REFCOUNT_ADD_OVF,
++ REFCOUNT_ADD_UAF,
++ REFCOUNT_SUB_UAF,
++ REFCOUNT_DEC_LEAK,
++};
++
++void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t);
++
++/**
++ * refcount_set - set a refcount's value
++ * @r: the refcount
++ * @n: value to which the refcount will be set
++ */
++static inline void refcount_set(refcount_t *r, int n)
++{
++ atomic_set(&r->refs, n);
++}
++
++/**
++ * refcount_read - get a refcount's value
++ * @r: the refcount
++ *
++ * Return: the refcount's value
++ */
++static inline unsigned int refcount_read(const refcount_t *r)
++{
++ return atomic_read(&r->refs);
++}
++
+ /**
+ * refcount_add_not_zero - add a value to a refcount unless it is 0
+ * @i: the value to add to the refcount
+@@ -298,46 +296,6 @@ static inline void refcount_dec(refcount_t *r)
+ if (unlikely(atomic_fetch_sub_release(1, &r->refs) <= 1))
+ refcount_warn_saturate(r, REFCOUNT_DEC_LEAK);
+ }
+-#else /* CONFIG_REFCOUNT_FULL */
+-# ifdef CONFIG_ARCH_HAS_REFCOUNT
+-# include <asm/refcount.h>
+-# else
+-static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+-{
+- return atomic_add_unless(&r->refs, i, 0);
+-}
+-
+-static inline void refcount_add(int i, refcount_t *r)
+-{
+- atomic_add(i, &r->refs);
+-}
+-
+-static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
+-{
+- return atomic_add_unless(&r->refs, 1, 0);
+-}
+-
+-static inline void refcount_inc(refcount_t *r)
+-{
+- atomic_inc(&r->refs);
+-}
+-
+-static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
+-{
+- return atomic_sub_and_test(i, &r->refs);
+-}
+-
+-static inline __must_check bool refcount_dec_and_test(refcount_t *r)
+-{
+- return atomic_dec_and_test(&r->refs);
+-}
+-
+-static inline void refcount_dec(refcount_t *r)
+-{
+- atomic_dec(&r->refs);
+-}
+-# endif /* !CONFIG_ARCH_HAS_REFCOUNT */
+-#endif /* !CONFIG_REFCOUNT_FULL */
+
+ extern __must_check bool refcount_dec_if_one(refcount_t *r);
+ extern __must_check bool refcount_dec_not_one(refcount_t *r);
+diff --git a/lib/refcount.c b/lib/refcount.c
+index 8b7e249c0e10..ebac8b7d15a7 100644
+--- a/lib/refcount.c
++++ b/lib/refcount.c
+@@ -1,6 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0
+ /*
+- * Out-of-line refcount functions common to all refcount implementations.
++ * Out-of-line refcount functions.
+ */
+
+ #include <linux/mutex.h>
+--
+2.35.1
+
--- /dev/null
+From d716a7dfb49329ead7158ad0e2e068e13e64c850 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:59 +0000
+Subject: locking/refcount: Consolidate REFCOUNT_{MAX,SATURATED} definitions
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit 65b008552469f1c37f5e06e0016924502e40b4f5 ]
+
+The definitions of REFCOUNT_MAX and REFCOUNT_SATURATED are the same,
+regardless of CONFIG_REFCOUNT_FULL, so consolidate them into a single
+pair of definitions.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-8-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/refcount.h | 9 ++-------
+ 1 file changed, 2 insertions(+), 7 deletions(-)
+
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index 1cd0a876a789..757d4630115c 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -22,6 +22,8 @@ typedef struct refcount_struct {
+ } refcount_t;
+
+ #define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), }
++#define REFCOUNT_MAX INT_MAX
++#define REFCOUNT_SATURATED (INT_MIN / 2)
+
+ enum refcount_saturation_type {
+ REFCOUNT_ADD_NOT_ZERO_OVF,
+@@ -57,9 +59,6 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ #ifdef CONFIG_REFCOUNT_FULL
+ #include <linux/bug.h>
+
+-#define REFCOUNT_MAX INT_MAX
+-#define REFCOUNT_SATURATED (INT_MIN / 2)
+-
+ /*
+ * Variant of atomic_t specialized for reference counts.
+ *
+@@ -300,10 +299,6 @@ static inline void refcount_dec(refcount_t *r)
+ refcount_warn_saturate(r, REFCOUNT_DEC_LEAK);
+ }
+ #else /* CONFIG_REFCOUNT_FULL */
+-
+-#define REFCOUNT_MAX INT_MAX
+-#define REFCOUNT_SATURATED (INT_MIN / 2)
+-
+ # ifdef CONFIG_ARCH_HAS_REFCOUNT
+ # include <asm/refcount.h>
+ # else
+--
+2.35.1
+
--- /dev/null
+From bb7dc070751272c95bce37f086106d0d01b13be8 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:53 +0000
+Subject: locking/refcount: Define constants for saturation and max refcount
+ values
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit 23e6b169c9917fbd77534f8c5f378cb073f548bd ]
+
+The REFCOUNT_FULL implementation uses a different saturation point than
+the x86 implementation, which means that the shared refcount code in
+lib/refcount.c (e.g. refcount_dec_not_one()) needs to be aware of the
+difference.
+
+Rather than duplicate the definitions from the lkdtm driver, instead
+move them into <linux/refcount.h> and update all references accordingly.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-2-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ drivers/misc/lkdtm/refcount.c | 8 --------
+ include/linux/refcount.h | 10 +++++++++-
+ lib/refcount.c | 37 +++++++++++++++++++----------------
+ 3 files changed, 29 insertions(+), 26 deletions(-)
+
+diff --git a/drivers/misc/lkdtm/refcount.c b/drivers/misc/lkdtm/refcount.c
+index 0a146b32da13..abf3b7c1f686 100644
+--- a/drivers/misc/lkdtm/refcount.c
++++ b/drivers/misc/lkdtm/refcount.c
+@@ -6,14 +6,6 @@
+ #include "lkdtm.h"
+ #include <linux/refcount.h>
+
+-#ifdef CONFIG_REFCOUNT_FULL
+-#define REFCOUNT_MAX (UINT_MAX - 1)
+-#define REFCOUNT_SATURATED UINT_MAX
+-#else
+-#define REFCOUNT_MAX INT_MAX
+-#define REFCOUNT_SATURATED (INT_MIN / 2)
+-#endif
+-
+ static void overflow_check(refcount_t *ref)
+ {
+ switch (refcount_read(ref)) {
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index e28cce21bad6..79f62e8d2256 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -4,6 +4,7 @@
+
+ #include <linux/atomic.h>
+ #include <linux/compiler.h>
++#include <linux/limits.h>
+ #include <linux/spinlock_types.h>
+
+ struct mutex;
+@@ -12,7 +13,7 @@ struct mutex;
+ * struct refcount_t - variant of atomic_t specialized for reference counts
+ * @refs: atomic_t counter field
+ *
+- * The counter saturates at UINT_MAX and will not move once
++ * The counter saturates at REFCOUNT_SATURATED and will not move once
+ * there. This avoids wrapping the counter and causing 'spurious'
+ * use-after-free bugs.
+ */
+@@ -56,6 +57,9 @@ extern void refcount_dec_checked(refcount_t *r);
+
+ #ifdef CONFIG_REFCOUNT_FULL
+
++#define REFCOUNT_MAX (UINT_MAX - 1)
++#define REFCOUNT_SATURATED UINT_MAX
++
+ #define refcount_add_not_zero refcount_add_not_zero_checked
+ #define refcount_add refcount_add_checked
+
+@@ -68,6 +72,10 @@ extern void refcount_dec_checked(refcount_t *r);
+ #define refcount_dec refcount_dec_checked
+
+ #else
++
++#define REFCOUNT_MAX INT_MAX
++#define REFCOUNT_SATURATED (INT_MIN / 2)
++
+ # ifdef CONFIG_ARCH_HAS_REFCOUNT
+ # include <asm/refcount.h>
+ # else
+diff --git a/lib/refcount.c b/lib/refcount.c
+index 6e904af0fb3e..48b78a423d7d 100644
+--- a/lib/refcount.c
++++ b/lib/refcount.c
+@@ -5,8 +5,8 @@
+ * The interface matches the atomic_t interface (to aid in porting) but only
+ * provides the few functions one should use for reference counting.
+ *
+- * It differs in that the counter saturates at UINT_MAX and will not move once
+- * there. This avoids wrapping the counter and causing 'spurious'
++ * It differs in that the counter saturates at REFCOUNT_SATURATED and will not
++ * move once there. This avoids wrapping the counter and causing 'spurious'
+ * use-after-free issues.
+ *
+ * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
+@@ -48,7 +48,7 @@
+ * @i: the value to add to the refcount
+ * @r: the refcount
+ *
+- * Will saturate at UINT_MAX and WARN.
++ * Will saturate at REFCOUNT_SATURATED and WARN.
+ *
+ * Provides no memory ordering, it is assumed the caller has guaranteed the
+ * object memory to be stable (RCU, etc.). It does provide a control dependency
+@@ -69,16 +69,17 @@ bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r)
+ if (!val)
+ return false;
+
+- if (unlikely(val == UINT_MAX))
++ if (unlikely(val == REFCOUNT_SATURATED))
+ return true;
+
+ new = val + i;
+ if (new < val)
+- new = UINT_MAX;
++ new = REFCOUNT_SATURATED;
+
+ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
+
+- WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
++ WARN_ONCE(new == REFCOUNT_SATURATED,
++ "refcount_t: saturated; leaking memory.\n");
+
+ return true;
+ }
+@@ -89,7 +90,7 @@ EXPORT_SYMBOL(refcount_add_not_zero_checked);
+ * @i: the value to add to the refcount
+ * @r: the refcount
+ *
+- * Similar to atomic_add(), but will saturate at UINT_MAX and WARN.
++ * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN.
+ *
+ * Provides no memory ordering, it is assumed the caller has guaranteed the
+ * object memory to be stable (RCU, etc.). It does provide a control dependency
+@@ -110,7 +111,8 @@ EXPORT_SYMBOL(refcount_add_checked);
+ * refcount_inc_not_zero_checked - increment a refcount unless it is 0
+ * @r: the refcount to increment
+ *
+- * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN.
++ * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED
++ * and WARN.
+ *
+ * Provides no memory ordering, it is assumed the caller has guaranteed the
+ * object memory to be stable (RCU, etc.). It does provide a control dependency
+@@ -133,7 +135,8 @@ bool refcount_inc_not_zero_checked(refcount_t *r)
+
+ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
+
+- WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
++ WARN_ONCE(new == REFCOUNT_SATURATED,
++ "refcount_t: saturated; leaking memory.\n");
+
+ return true;
+ }
+@@ -143,7 +146,7 @@ EXPORT_SYMBOL(refcount_inc_not_zero_checked);
+ * refcount_inc_checked - increment a refcount
+ * @r: the refcount to increment
+ *
+- * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN.
++ * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN.
+ *
+ * Provides no memory ordering, it is assumed the caller already has a
+ * reference on the object.
+@@ -164,7 +167,7 @@ EXPORT_SYMBOL(refcount_inc_checked);
+ *
+ * Similar to atomic_dec_and_test(), but it will WARN, return false and
+ * ultimately leak on underflow and will fail to decrement when saturated
+- * at UINT_MAX.
++ * at REFCOUNT_SATURATED.
+ *
+ * Provides release memory ordering, such that prior loads and stores are done
+ * before, and provides an acquire ordering on success such that free()
+@@ -182,7 +185,7 @@ bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r)
+ unsigned int new, val = atomic_read(&r->refs);
+
+ do {
+- if (unlikely(val == UINT_MAX))
++ if (unlikely(val == REFCOUNT_SATURATED))
+ return false;
+
+ new = val - i;
+@@ -207,7 +210,7 @@ EXPORT_SYMBOL(refcount_sub_and_test_checked);
+ * @r: the refcount
+ *
+ * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
+- * decrement when saturated at UINT_MAX.
++ * decrement when saturated at REFCOUNT_SATURATED.
+ *
+ * Provides release memory ordering, such that prior loads and stores are done
+ * before, and provides an acquire ordering on success such that free()
+@@ -226,7 +229,7 @@ EXPORT_SYMBOL(refcount_dec_and_test_checked);
+ * @r: the refcount
+ *
+ * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
+- * when saturated at UINT_MAX.
++ * when saturated at REFCOUNT_SATURATED.
+ *
+ * Provides release memory ordering, such that prior loads and stores are done
+ * before.
+@@ -277,7 +280,7 @@ bool refcount_dec_not_one(refcount_t *r)
+ unsigned int new, val = atomic_read(&r->refs);
+
+ do {
+- if (unlikely(val == UINT_MAX))
++ if (unlikely(val == REFCOUNT_SATURATED))
+ return true;
+
+ if (val == 1)
+@@ -302,7 +305,7 @@ EXPORT_SYMBOL(refcount_dec_not_one);
+ * @lock: the mutex to be locked
+ *
+ * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail
+- * to decrement when saturated at UINT_MAX.
++ * to decrement when saturated at REFCOUNT_SATURATED.
+ *
+ * Provides release memory ordering, such that prior loads and stores are done
+ * before, and provides a control dependency such that free() must come after.
+@@ -333,7 +336,7 @@ EXPORT_SYMBOL(refcount_dec_and_mutex_lock);
+ * @lock: the spinlock to be locked
+ *
+ * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to
+- * decrement when saturated at UINT_MAX.
++ * decrement when saturated at REFCOUNT_SATURATED.
+ *
+ * Provides release memory ordering, such that prior loads and stores are done
+ * before, and provides a control dependency such that free() must come after.
+--
+2.35.1
+
--- /dev/null
+From db84554aa58fd3d15d6299589bd50987c70612b7 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:54 +0000
+Subject: locking/refcount: Ensure integer operands are treated as signed
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit 97a1420adf0cdf0cf6f41bab0b2acf658c96b94b ]
+
+In preparation for changing the saturation point of REFCOUNT_FULL to
+INT_MIN/2, change the type of integer operands passed into the API
+from 'unsigned int' to 'int' so that we can avoid casting during
+comparisons when we don't want to fall foul of C integral conversion
+rules for signed and unsigned types.
+
+Since the kernel is compiled with '-fno-strict-overflow', we don't need
+to worry about the UB introduced by signed overflow here. Furthermore,
+we're already making heavy use of the atomic_t API, which operates
+exclusively on signed types.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-3-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/refcount.h | 14 +++++++-------
+ lib/refcount.c | 6 +++---
+ 2 files changed, 10 insertions(+), 10 deletions(-)
+
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index 79f62e8d2256..89066a1471dd 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -28,7 +28,7 @@ typedef struct refcount_struct {
+ * @r: the refcount
+ * @n: value to which the refcount will be set
+ */
+-static inline void refcount_set(refcount_t *r, unsigned int n)
++static inline void refcount_set(refcount_t *r, int n)
+ {
+ atomic_set(&r->refs, n);
+ }
+@@ -44,13 +44,13 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ return atomic_read(&r->refs);
+ }
+
+-extern __must_check bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r);
+-extern void refcount_add_checked(unsigned int i, refcount_t *r);
++extern __must_check bool refcount_add_not_zero_checked(int i, refcount_t *r);
++extern void refcount_add_checked(int i, refcount_t *r);
+
+ extern __must_check bool refcount_inc_not_zero_checked(refcount_t *r);
+ extern void refcount_inc_checked(refcount_t *r);
+
+-extern __must_check bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r);
++extern __must_check bool refcount_sub_and_test_checked(int i, refcount_t *r);
+
+ extern __must_check bool refcount_dec_and_test_checked(refcount_t *r);
+ extern void refcount_dec_checked(refcount_t *r);
+@@ -79,12 +79,12 @@ extern void refcount_dec_checked(refcount_t *r);
+ # ifdef CONFIG_ARCH_HAS_REFCOUNT
+ # include <asm/refcount.h>
+ # else
+-static inline __must_check bool refcount_add_not_zero(unsigned int i, refcount_t *r)
++static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+ {
+ return atomic_add_unless(&r->refs, i, 0);
+ }
+
+-static inline void refcount_add(unsigned int i, refcount_t *r)
++static inline void refcount_add(int i, refcount_t *r)
+ {
+ atomic_add(i, &r->refs);
+ }
+@@ -99,7 +99,7 @@ static inline void refcount_inc(refcount_t *r)
+ atomic_inc(&r->refs);
+ }
+
+-static inline __must_check bool refcount_sub_and_test(unsigned int i, refcount_t *r)
++static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
+ {
+ return atomic_sub_and_test(i, &r->refs);
+ }
+diff --git a/lib/refcount.c b/lib/refcount.c
+index 48b78a423d7d..719b0bc42ab1 100644
+--- a/lib/refcount.c
++++ b/lib/refcount.c
+@@ -61,7 +61,7 @@
+ *
+ * Return: false if the passed refcount is 0, true otherwise
+ */
+-bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r)
++bool refcount_add_not_zero_checked(int i, refcount_t *r)
+ {
+ unsigned int new, val = atomic_read(&r->refs);
+
+@@ -101,7 +101,7 @@ EXPORT_SYMBOL(refcount_add_not_zero_checked);
+ * cases, refcount_inc(), or one of its variants, should instead be used to
+ * increment a reference count.
+ */
+-void refcount_add_checked(unsigned int i, refcount_t *r)
++void refcount_add_checked(int i, refcount_t *r)
+ {
+ WARN_ONCE(!refcount_add_not_zero_checked(i, r), "refcount_t: addition on 0; use-after-free.\n");
+ }
+@@ -180,7 +180,7 @@ EXPORT_SYMBOL(refcount_inc_checked);
+ *
+ * Return: true if the resulting refcount is 0, false otherwise
+ */
+-bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r)
++bool refcount_sub_and_test_checked(int i, refcount_t *r)
+ {
+ unsigned int new, val = atomic_read(&r->refs);
+
+--
+2.35.1
+
--- /dev/null
+From 6eaed59065030285985ed945780aacb7dec937cd Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:57 +0000
+Subject: locking/refcount: Improve performance of generic REFCOUNT_FULL code
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit dcb786493f3e48da3272b710028d42ec608cfda1 ]
+
+Rewrite the generic REFCOUNT_FULL implementation so that the saturation
+point is moved to INT_MIN / 2. This allows us to defer the sanity checks
+until after the atomic operation, which removes many uses of cmpxchg()
+in favour of atomic_fetch_{add,sub}().
+
+Some crude perf results obtained from lkdtm show substantially less
+overhead, despite the checking:
+
+ $ perf stat -r 3 -B -- echo {ATOMIC,REFCOUNT}_TIMING >/sys/kernel/debug/provoke-crash/DIRECT
+
+ # arm64
+ ATOMIC_TIMING: 46.50451 +- 0.00134 seconds time elapsed ( +- 0.00% )
+ REFCOUNT_TIMING (REFCOUNT_FULL, mainline): 77.57522 +- 0.00982 seconds time elapsed ( +- 0.01% )
+ REFCOUNT_TIMING (REFCOUNT_FULL, this series): 48.7181 +- 0.0256 seconds time elapsed ( +- 0.05% )
+
+ # x86
+ ATOMIC_TIMING: 31.6225 +- 0.0776 seconds time elapsed ( +- 0.25% )
+ REFCOUNT_TIMING (!REFCOUNT_FULL, mainline/x86 asm): 31.6689 +- 0.0901 seconds time elapsed ( +- 0.28% )
+ REFCOUNT_TIMING (REFCOUNT_FULL, mainline): 53.203 +- 0.138 seconds time elapsed ( +- 0.26% )
+ REFCOUNT_TIMING (REFCOUNT_FULL, this series): 31.7408 +- 0.0486 seconds time elapsed ( +- 0.15% )
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Tested-by: Jan Glauber <jglauber@marvell.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-6-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/refcount.h | 131 ++++++++++++++++++++++-----------------
+ 1 file changed, 75 insertions(+), 56 deletions(-)
+
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index e719b5b1220e..e3b218d669ce 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -47,8 +47,8 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ #ifdef CONFIG_REFCOUNT_FULL
+ #include <linux/bug.h>
+
+-#define REFCOUNT_MAX (UINT_MAX - 1)
+-#define REFCOUNT_SATURATED UINT_MAX
++#define REFCOUNT_MAX INT_MAX
++#define REFCOUNT_SATURATED (INT_MIN / 2)
+
+ /*
+ * Variant of atomic_t specialized for reference counts.
+@@ -56,9 +56,47 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ * The interface matches the atomic_t interface (to aid in porting) but only
+ * provides the few functions one should use for reference counting.
+ *
+- * It differs in that the counter saturates at REFCOUNT_SATURATED and will not
+- * move once there. This avoids wrapping the counter and causing 'spurious'
+- * use-after-free issues.
++ * Saturation semantics
++ * ====================
++ *
++ * refcount_t differs from atomic_t in that the counter saturates at
++ * REFCOUNT_SATURATED and will not move once there. This avoids wrapping the
++ * counter and causing 'spurious' use-after-free issues. In order to avoid the
++ * cost associated with introducing cmpxchg() loops into all of the saturating
++ * operations, we temporarily allow the counter to take on an unchecked value
++ * and then explicitly set it to REFCOUNT_SATURATED on detecting that underflow
++ * or overflow has occurred. Although this is racy when multiple threads
++ * access the refcount concurrently, by placing REFCOUNT_SATURATED roughly
++ * equidistant from 0 and INT_MAX we minimise the scope for error:
++ *
++ * INT_MAX REFCOUNT_SATURATED UINT_MAX
++ * 0 (0x7fff_ffff) (0xc000_0000) (0xffff_ffff)
++ * +--------------------------------+----------------+----------------+
++ * <---------- bad value! ---------->
++ *
++ * (in a signed view of the world, the "bad value" range corresponds to
++ * a negative counter value).
++ *
++ * As an example, consider a refcount_inc() operation that causes the counter
++ * to overflow:
++ *
++ * int old = atomic_fetch_add_relaxed(r);
++ * // old is INT_MAX, refcount now INT_MIN (0x8000_0000)
++ * if (old < 0)
++ * atomic_set(r, REFCOUNT_SATURATED);
++ *
++ * If another thread also performs a refcount_inc() operation between the two
++ * atomic operations, then the count will continue to edge closer to 0. If it
++ * reaches a value of 1 before /any/ of the threads reset it to the saturated
++ * value, then a concurrent refcount_dec_and_test() may erroneously free the
++ * underlying object. Given the precise timing details involved with the
++ * round-robin scheduling of each thread manipulating the refcount and the need
++ * to hit the race multiple times in succession, there doesn't appear to be a
++ * practical avenue of attack even if using refcount_add() operations with
++ * larger increments.
++ *
++ * Memory ordering
++ * ===============
+ *
+ * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
+ * and provide only what is strictly required for refcounts.
+@@ -109,25 +147,19 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ */
+ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+ {
+- unsigned int new, val = atomic_read(&r->refs);
++ int old = refcount_read(r);
+
+ do {
+- if (!val)
+- return false;
+-
+- if (unlikely(val == REFCOUNT_SATURATED))
+- return true;
+-
+- new = val + i;
+- if (new < val)
+- new = REFCOUNT_SATURATED;
++ if (!old)
++ break;
++ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));
+
+- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
+-
+- WARN_ONCE(new == REFCOUNT_SATURATED,
+- "refcount_t: saturated; leaking memory.\n");
++ if (unlikely(old < 0 || old + i < 0)) {
++ refcount_set(r, REFCOUNT_SATURATED);
++ WARN_ONCE(1, "refcount_t: saturated; leaking memory.\n");
++ }
+
+- return true;
++ return old;
+ }
+
+ /**
+@@ -148,7 +180,13 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+ */
+ static inline void refcount_add(int i, refcount_t *r)
+ {
+- WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
++ int old = atomic_fetch_add_relaxed(i, &r->refs);
++
++ WARN_ONCE(!old, "refcount_t: addition on 0; use-after-free.\n");
++ if (unlikely(old <= 0 || old + i <= 0)) {
++ refcount_set(r, REFCOUNT_SATURATED);
++ WARN_ONCE(old, "refcount_t: saturated; leaking memory.\n");
++ }
+ }
+
+ /**
+@@ -166,23 +204,7 @@ static inline void refcount_add(int i, refcount_t *r)
+ */
+ static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
+ {
+- unsigned int new, val = atomic_read(&r->refs);
+-
+- do {
+- new = val + 1;
+-
+- if (!val)
+- return false;
+-
+- if (unlikely(!new))
+- return true;
+-
+- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
+-
+- WARN_ONCE(new == REFCOUNT_SATURATED,
+- "refcount_t: saturated; leaking memory.\n");
+-
+- return true;
++ return refcount_add_not_zero(1, r);
+ }
+
+ /**
+@@ -199,7 +221,7 @@ static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
+ */
+ static inline void refcount_inc(refcount_t *r)
+ {
+- WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
++ refcount_add(1, r);
+ }
+
+ /**
+@@ -224,26 +246,19 @@ static inline void refcount_inc(refcount_t *r)
+ */
+ static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
+ {
+- unsigned int new, val = atomic_read(&r->refs);
+-
+- do {
+- if (unlikely(val == REFCOUNT_SATURATED))
+- return false;
++ int old = atomic_fetch_sub_release(i, &r->refs);
+
+- new = val - i;
+- if (new > val) {
+- WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
+- return false;
+- }
+-
+- } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
+-
+- if (!new) {
++ if (old == i) {
+ smp_acquire__after_ctrl_dep();
+ return true;
+ }
+- return false;
+
++ if (unlikely(old < 0 || old - i < 0)) {
++ refcount_set(r, REFCOUNT_SATURATED);
++ WARN_ONCE(1, "refcount_t: underflow; use-after-free.\n");
++ }
++
++ return false;
+ }
+
+ /**
+@@ -276,9 +291,13 @@ static inline __must_check bool refcount_dec_and_test(refcount_t *r)
+ */
+ static inline void refcount_dec(refcount_t *r)
+ {
+- WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
+-}
++ int old = atomic_fetch_sub_release(1, &r->refs);
+
++ if (unlikely(old <= 1)) {
++ refcount_set(r, REFCOUNT_SATURATED);
++ WARN_ONCE(1, "refcount_t: decrement hit 0; leaking memory.\n");
++ }
++}
+ #else /* CONFIG_REFCOUNT_FULL */
+
+ #define REFCOUNT_MAX INT_MAX
+--
+2.35.1
+
--- /dev/null
+From 4692c5514cef668722dfaeab60870fe90ec9f978 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:58 +0000
+Subject: locking/refcount: Move saturation warnings out of line
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit 1eb085d94256aaa69b00cf5a86e3c5f5bb2bc460 ]
+
+Having the refcount saturation and warnings inline bloats the text,
+despite the fact that these paths should never be executed in normal
+operation.
+
+Move the refcount saturation and warnings out of line to reduce the
+image size when refcount_t checking is enabled. Relative to an x86_64
+defconfig, the sizes reported by bloat-o-meter are:
+
+ # defconfig+REFCOUNT_FULL, inline saturation (i.e. before this patch)
+ Total: Before=14762076, After=14915442, chg +1.04%
+
+ # defconfig+REFCOUNT_FULL, out-of-line saturation (i.e. after this patch)
+ Total: Before=14762076, After=14835497, chg +0.50%
+
+A side-effect of this change is that we now only get one warning per
+refcount saturation type, rather than one per problematic call-site.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-7-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/refcount.h | 39 ++++++++++++++++++++-------------------
+ lib/refcount.c | 28 ++++++++++++++++++++++++++++
+ 2 files changed, 48 insertions(+), 19 deletions(-)
+
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index e3b218d669ce..1cd0a876a789 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -23,6 +23,16 @@ typedef struct refcount_struct {
+
+ #define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), }
+
++enum refcount_saturation_type {
++ REFCOUNT_ADD_NOT_ZERO_OVF,
++ REFCOUNT_ADD_OVF,
++ REFCOUNT_ADD_UAF,
++ REFCOUNT_SUB_UAF,
++ REFCOUNT_DEC_LEAK,
++};
++
++void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t);
++
+ /**
+ * refcount_set - set a refcount's value
+ * @r: the refcount
+@@ -154,10 +164,8 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+ break;
+ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));
+
+- if (unlikely(old < 0 || old + i < 0)) {
+- refcount_set(r, REFCOUNT_SATURATED);
+- WARN_ONCE(1, "refcount_t: saturated; leaking memory.\n");
+- }
++ if (unlikely(old < 0 || old + i < 0))
++ refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF);
+
+ return old;
+ }
+@@ -182,11 +190,10 @@ static inline void refcount_add(int i, refcount_t *r)
+ {
+ int old = atomic_fetch_add_relaxed(i, &r->refs);
+
+- WARN_ONCE(!old, "refcount_t: addition on 0; use-after-free.\n");
+- if (unlikely(old <= 0 || old + i <= 0)) {
+- refcount_set(r, REFCOUNT_SATURATED);
+- WARN_ONCE(old, "refcount_t: saturated; leaking memory.\n");
+- }
++ if (unlikely(!old))
++ refcount_warn_saturate(r, REFCOUNT_ADD_UAF);
++ else if (unlikely(old < 0 || old + i < 0))
++ refcount_warn_saturate(r, REFCOUNT_ADD_OVF);
+ }
+
+ /**
+@@ -253,10 +260,8 @@ static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
+ return true;
+ }
+
+- if (unlikely(old < 0 || old - i < 0)) {
+- refcount_set(r, REFCOUNT_SATURATED);
+- WARN_ONCE(1, "refcount_t: underflow; use-after-free.\n");
+- }
++ if (unlikely(old < 0 || old - i < 0))
++ refcount_warn_saturate(r, REFCOUNT_SUB_UAF);
+
+ return false;
+ }
+@@ -291,12 +296,8 @@ static inline __must_check bool refcount_dec_and_test(refcount_t *r)
+ */
+ static inline void refcount_dec(refcount_t *r)
+ {
+- int old = atomic_fetch_sub_release(1, &r->refs);
+-
+- if (unlikely(old <= 1)) {
+- refcount_set(r, REFCOUNT_SATURATED);
+- WARN_ONCE(1, "refcount_t: decrement hit 0; leaking memory.\n");
+- }
++ if (unlikely(atomic_fetch_sub_release(1, &r->refs) <= 1))
++ refcount_warn_saturate(r, REFCOUNT_DEC_LEAK);
+ }
+ #else /* CONFIG_REFCOUNT_FULL */
+
+diff --git a/lib/refcount.c b/lib/refcount.c
+index 3a534fbebdcc..8b7e249c0e10 100644
+--- a/lib/refcount.c
++++ b/lib/refcount.c
+@@ -8,6 +8,34 @@
+ #include <linux/spinlock.h>
+ #include <linux/bug.h>
+
++#define REFCOUNT_WARN(str) WARN_ONCE(1, "refcount_t: " str ".\n")
++
++void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t)
++{
++ refcount_set(r, REFCOUNT_SATURATED);
++
++ switch (t) {
++ case REFCOUNT_ADD_NOT_ZERO_OVF:
++ REFCOUNT_WARN("saturated; leaking memory");
++ break;
++ case REFCOUNT_ADD_OVF:
++ REFCOUNT_WARN("saturated; leaking memory");
++ break;
++ case REFCOUNT_ADD_UAF:
++ REFCOUNT_WARN("addition on 0; use-after-free");
++ break;
++ case REFCOUNT_SUB_UAF:
++ REFCOUNT_WARN("underflow; use-after-free");
++ break;
++ case REFCOUNT_DEC_LEAK:
++ REFCOUNT_WARN("decrement hit 0; leaking memory");
++ break;
++ default:
++ REFCOUNT_WARN("unknown saturation event!?");
++ }
++}
++EXPORT_SYMBOL(refcount_warn_saturate);
++
+ /**
+ * refcount_dec_if_one - decrement a refcount if it is 1
+ * @r: the refcount
+--
+2.35.1
+
--- /dev/null
+From 27c24b3cffb5963450ebc988eee4780cce1961ea Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:56 +0000
+Subject: locking/refcount: Move the bulk of the REFCOUNT_FULL implementation
+ into the <linux/refcount.h> header
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit 77e9971c79c29542ab7dd4140f9343bf2ff36158 ]
+
+In an effort to improve performance of the REFCOUNT_FULL implementation,
+move the bulk of its functions into linux/refcount.h. This allows them
+to be inlined in the same way as if they had been provided via
+CONFIG_ARCH_HAS_REFCOUNT.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-5-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/refcount.h | 237 ++++++++++++++++++++++++++++++++++++--
+ lib/refcount.c | 238 +--------------------------------------
+ 2 files changed, 229 insertions(+), 246 deletions(-)
+
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index edd505d1a23b..e719b5b1220e 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -45,22 +45,241 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ }
+
+ #ifdef CONFIG_REFCOUNT_FULL
++#include <linux/bug.h>
+
+ #define REFCOUNT_MAX (UINT_MAX - 1)
+ #define REFCOUNT_SATURATED UINT_MAX
+
+-extern __must_check bool refcount_add_not_zero(int i, refcount_t *r);
+-extern void refcount_add(int i, refcount_t *r);
++/*
++ * Variant of atomic_t specialized for reference counts.
++ *
++ * The interface matches the atomic_t interface (to aid in porting) but only
++ * provides the few functions one should use for reference counting.
++ *
++ * It differs in that the counter saturates at REFCOUNT_SATURATED and will not
++ * move once there. This avoids wrapping the counter and causing 'spurious'
++ * use-after-free issues.
++ *
++ * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
++ * and provide only what is strictly required for refcounts.
++ *
++ * The increments are fully relaxed; these will not provide ordering. The
++ * rationale is that whatever is used to obtain the object we're increasing the
++ * reference count on will provide the ordering. For locked data structures,
++ * its the lock acquire, for RCU/lockless data structures its the dependent
++ * load.
++ *
++ * Do note that inc_not_zero() provides a control dependency which will order
++ * future stores against the inc, this ensures we'll never modify the object
++ * if we did not in fact acquire a reference.
++ *
++ * The decrements will provide release order, such that all the prior loads and
++ * stores will be issued before, it also provides a control dependency, which
++ * will order us against the subsequent free().
++ *
++ * The control dependency is against the load of the cmpxchg (ll/sc) that
++ * succeeded. This means the stores aren't fully ordered, but this is fine
++ * because the 1->0 transition indicates no concurrency.
++ *
++ * Note that the allocator is responsible for ordering things between free()
++ * and alloc().
++ *
++ * The decrements dec_and_test() and sub_and_test() also provide acquire
++ * ordering on success.
++ *
++ */
++
++/**
++ * refcount_add_not_zero - add a value to a refcount unless it is 0
++ * @i: the value to add to the refcount
++ * @r: the refcount
++ *
++ * Will saturate at REFCOUNT_SATURATED and WARN.
++ *
++ * Provides no memory ordering, it is assumed the caller has guaranteed the
++ * object memory to be stable (RCU, etc.). It does provide a control dependency
++ * and thereby orders future stores. See the comment on top.
++ *
++ * Use of this function is not recommended for the normal reference counting
++ * use case in which references are taken and released one at a time. In these
++ * cases, refcount_inc(), or one of its variants, should instead be used to
++ * increment a reference count.
++ *
++ * Return: false if the passed refcount is 0, true otherwise
++ */
++static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
++{
++ unsigned int new, val = atomic_read(&r->refs);
++
++ do {
++ if (!val)
++ return false;
++
++ if (unlikely(val == REFCOUNT_SATURATED))
++ return true;
++
++ new = val + i;
++ if (new < val)
++ new = REFCOUNT_SATURATED;
++
++ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
++
++ WARN_ONCE(new == REFCOUNT_SATURATED,
++ "refcount_t: saturated; leaking memory.\n");
++
++ return true;
++}
++
++/**
++ * refcount_add - add a value to a refcount
++ * @i: the value to add to the refcount
++ * @r: the refcount
++ *
++ * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN.
++ *
++ * Provides no memory ordering, it is assumed the caller has guaranteed the
++ * object memory to be stable (RCU, etc.). It does provide a control dependency
++ * and thereby orders future stores. See the comment on top.
++ *
++ * Use of this function is not recommended for the normal reference counting
++ * use case in which references are taken and released one at a time. In these
++ * cases, refcount_inc(), or one of its variants, should instead be used to
++ * increment a reference count.
++ */
++static inline void refcount_add(int i, refcount_t *r)
++{
++ WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
++}
++
++/**
++ * refcount_inc_not_zero - increment a refcount unless it is 0
++ * @r: the refcount to increment
++ *
++ * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED
++ * and WARN.
++ *
++ * Provides no memory ordering, it is assumed the caller has guaranteed the
++ * object memory to be stable (RCU, etc.). It does provide a control dependency
++ * and thereby orders future stores. See the comment on top.
++ *
++ * Return: true if the increment was successful, false otherwise
++ */
++static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
++{
++ unsigned int new, val = atomic_read(&r->refs);
++
++ do {
++ new = val + 1;
+
+-extern __must_check bool refcount_inc_not_zero(refcount_t *r);
+-extern void refcount_inc(refcount_t *r);
++ if (!val)
++ return false;
+
+-extern __must_check bool refcount_sub_and_test(int i, refcount_t *r);
++ if (unlikely(!new))
++ return true;
+
+-extern __must_check bool refcount_dec_and_test(refcount_t *r);
+-extern void refcount_dec(refcount_t *r);
++ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
++
++ WARN_ONCE(new == REFCOUNT_SATURATED,
++ "refcount_t: saturated; leaking memory.\n");
++
++ return true;
++}
++
++/**
++ * refcount_inc - increment a refcount
++ * @r: the refcount to increment
++ *
++ * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN.
++ *
++ * Provides no memory ordering, it is assumed the caller already has a
++ * reference on the object.
++ *
++ * Will WARN if the refcount is 0, as this represents a possible use-after-free
++ * condition.
++ */
++static inline void refcount_inc(refcount_t *r)
++{
++ WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
++}
++
++/**
++ * refcount_sub_and_test - subtract from a refcount and test if it is 0
++ * @i: amount to subtract from the refcount
++ * @r: the refcount
++ *
++ * Similar to atomic_dec_and_test(), but it will WARN, return false and
++ * ultimately leak on underflow and will fail to decrement when saturated
++ * at REFCOUNT_SATURATED.
++ *
++ * Provides release memory ordering, such that prior loads and stores are done
++ * before, and provides an acquire ordering on success such that free()
++ * must come after.
++ *
++ * Use of this function is not recommended for the normal reference counting
++ * use case in which references are taken and released one at a time. In these
++ * cases, refcount_dec(), or one of its variants, should instead be used to
++ * decrement a reference count.
++ *
++ * Return: true if the resulting refcount is 0, false otherwise
++ */
++static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
++{
++ unsigned int new, val = atomic_read(&r->refs);
++
++ do {
++ if (unlikely(val == REFCOUNT_SATURATED))
++ return false;
++
++ new = val - i;
++ if (new > val) {
++ WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
++ return false;
++ }
++
++ } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
++
++ if (!new) {
++ smp_acquire__after_ctrl_dep();
++ return true;
++ }
++ return false;
++
++}
++
++/**
++ * refcount_dec_and_test - decrement a refcount and test if it is 0
++ * @r: the refcount
++ *
++ * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
++ * decrement when saturated at REFCOUNT_SATURATED.
++ *
++ * Provides release memory ordering, such that prior loads and stores are done
++ * before, and provides an acquire ordering on success such that free()
++ * must come after.
++ *
++ * Return: true if the resulting refcount is 0, false otherwise
++ */
++static inline __must_check bool refcount_dec_and_test(refcount_t *r)
++{
++ return refcount_sub_and_test(1, r);
++}
++
++/**
++ * refcount_dec - decrement a refcount
++ * @r: the refcount
++ *
++ * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
++ * when saturated at REFCOUNT_SATURATED.
++ *
++ * Provides release memory ordering, such that prior loads and stores are done
++ * before.
++ */
++static inline void refcount_dec(refcount_t *r)
++{
++ WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
++}
+
+-#else
++#else /* CONFIG_REFCOUNT_FULL */
+
+ #define REFCOUNT_MAX INT_MAX
+ #define REFCOUNT_SATURATED (INT_MIN / 2)
+@@ -103,7 +322,7 @@ static inline void refcount_dec(refcount_t *r)
+ atomic_dec(&r->refs);
+ }
+ # endif /* !CONFIG_ARCH_HAS_REFCOUNT */
+-#endif /* CONFIG_REFCOUNT_FULL */
++#endif /* !CONFIG_REFCOUNT_FULL */
+
+ extern __must_check bool refcount_dec_if_one(refcount_t *r);
+ extern __must_check bool refcount_dec_not_one(refcount_t *r);
+diff --git a/lib/refcount.c b/lib/refcount.c
+index a2f670998cee..3a534fbebdcc 100644
+--- a/lib/refcount.c
++++ b/lib/refcount.c
+@@ -1,41 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0
+ /*
+- * Variant of atomic_t specialized for reference counts.
+- *
+- * The interface matches the atomic_t interface (to aid in porting) but only
+- * provides the few functions one should use for reference counting.
+- *
+- * It differs in that the counter saturates at REFCOUNT_SATURATED and will not
+- * move once there. This avoids wrapping the counter and causing 'spurious'
+- * use-after-free issues.
+- *
+- * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
+- * and provide only what is strictly required for refcounts.
+- *
+- * The increments are fully relaxed; these will not provide ordering. The
+- * rationale is that whatever is used to obtain the object we're increasing the
+- * reference count on will provide the ordering. For locked data structures,
+- * its the lock acquire, for RCU/lockless data structures its the dependent
+- * load.
+- *
+- * Do note that inc_not_zero() provides a control dependency which will order
+- * future stores against the inc, this ensures we'll never modify the object
+- * if we did not in fact acquire a reference.
+- *
+- * The decrements will provide release order, such that all the prior loads and
+- * stores will be issued before, it also provides a control dependency, which
+- * will order us against the subsequent free().
+- *
+- * The control dependency is against the load of the cmpxchg (ll/sc) that
+- * succeeded. This means the stores aren't fully ordered, but this is fine
+- * because the 1->0 transition indicates no concurrency.
+- *
+- * Note that the allocator is responsible for ordering things between free()
+- * and alloc().
+- *
+- * The decrements dec_and_test() and sub_and_test() also provide acquire
+- * ordering on success.
+- *
++ * Out-of-line refcount functions common to all refcount implementations.
+ */
+
+ #include <linux/mutex.h>
+@@ -43,207 +8,6 @@
+ #include <linux/spinlock.h>
+ #include <linux/bug.h>
+
+-#ifdef CONFIG_REFCOUNT_FULL
+-
+-/**
+- * refcount_add_not_zero - add a value to a refcount unless it is 0
+- * @i: the value to add to the refcount
+- * @r: the refcount
+- *
+- * Will saturate at REFCOUNT_SATURATED and WARN.
+- *
+- * Provides no memory ordering, it is assumed the caller has guaranteed the
+- * object memory to be stable (RCU, etc.). It does provide a control dependency
+- * and thereby orders future stores. See the comment on top.
+- *
+- * Use of this function is not recommended for the normal reference counting
+- * use case in which references are taken and released one at a time. In these
+- * cases, refcount_inc(), or one of its variants, should instead be used to
+- * increment a reference count.
+- *
+- * Return: false if the passed refcount is 0, true otherwise
+- */
+-bool refcount_add_not_zero(int i, refcount_t *r)
+-{
+- unsigned int new, val = atomic_read(&r->refs);
+-
+- do {
+- if (!val)
+- return false;
+-
+- if (unlikely(val == REFCOUNT_SATURATED))
+- return true;
+-
+- new = val + i;
+- if (new < val)
+- new = REFCOUNT_SATURATED;
+-
+- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
+-
+- WARN_ONCE(new == REFCOUNT_SATURATED,
+- "refcount_t: saturated; leaking memory.\n");
+-
+- return true;
+-}
+-EXPORT_SYMBOL(refcount_add_not_zero);
+-
+-/**
+- * refcount_add - add a value to a refcount
+- * @i: the value to add to the refcount
+- * @r: the refcount
+- *
+- * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN.
+- *
+- * Provides no memory ordering, it is assumed the caller has guaranteed the
+- * object memory to be stable (RCU, etc.). It does provide a control dependency
+- * and thereby orders future stores. See the comment on top.
+- *
+- * Use of this function is not recommended for the normal reference counting
+- * use case in which references are taken and released one at a time. In these
+- * cases, refcount_inc(), or one of its variants, should instead be used to
+- * increment a reference count.
+- */
+-void refcount_add(int i, refcount_t *r)
+-{
+- WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
+-}
+-EXPORT_SYMBOL(refcount_add);
+-
+-/**
+- * refcount_inc_not_zero - increment a refcount unless it is 0
+- * @r: the refcount to increment
+- *
+- * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED
+- * and WARN.
+- *
+- * Provides no memory ordering, it is assumed the caller has guaranteed the
+- * object memory to be stable (RCU, etc.). It does provide a control dependency
+- * and thereby orders future stores. See the comment on top.
+- *
+- * Return: true if the increment was successful, false otherwise
+- */
+-bool refcount_inc_not_zero(refcount_t *r)
+-{
+- unsigned int new, val = atomic_read(&r->refs);
+-
+- do {
+- new = val + 1;
+-
+- if (!val)
+- return false;
+-
+- if (unlikely(!new))
+- return true;
+-
+- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
+-
+- WARN_ONCE(new == REFCOUNT_SATURATED,
+- "refcount_t: saturated; leaking memory.\n");
+-
+- return true;
+-}
+-EXPORT_SYMBOL(refcount_inc_not_zero);
+-
+-/**
+- * refcount_inc - increment a refcount
+- * @r: the refcount to increment
+- *
+- * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN.
+- *
+- * Provides no memory ordering, it is assumed the caller already has a
+- * reference on the object.
+- *
+- * Will WARN if the refcount is 0, as this represents a possible use-after-free
+- * condition.
+- */
+-void refcount_inc(refcount_t *r)
+-{
+- WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
+-}
+-EXPORT_SYMBOL(refcount_inc);
+-
+-/**
+- * refcount_sub_and_test - subtract from a refcount and test if it is 0
+- * @i: amount to subtract from the refcount
+- * @r: the refcount
+- *
+- * Similar to atomic_dec_and_test(), but it will WARN, return false and
+- * ultimately leak on underflow and will fail to decrement when saturated
+- * at REFCOUNT_SATURATED.
+- *
+- * Provides release memory ordering, such that prior loads and stores are done
+- * before, and provides an acquire ordering on success such that free()
+- * must come after.
+- *
+- * Use of this function is not recommended for the normal reference counting
+- * use case in which references are taken and released one at a time. In these
+- * cases, refcount_dec(), or one of its variants, should instead be used to
+- * decrement a reference count.
+- *
+- * Return: true if the resulting refcount is 0, false otherwise
+- */
+-bool refcount_sub_and_test(int i, refcount_t *r)
+-{
+- unsigned int new, val = atomic_read(&r->refs);
+-
+- do {
+- if (unlikely(val == REFCOUNT_SATURATED))
+- return false;
+-
+- new = val - i;
+- if (new > val) {
+- WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
+- return false;
+- }
+-
+- } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
+-
+- if (!new) {
+- smp_acquire__after_ctrl_dep();
+- return true;
+- }
+- return false;
+-
+-}
+-EXPORT_SYMBOL(refcount_sub_and_test);
+-
+-/**
+- * refcount_dec_and_test - decrement a refcount and test if it is 0
+- * @r: the refcount
+- *
+- * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
+- * decrement when saturated at REFCOUNT_SATURATED.
+- *
+- * Provides release memory ordering, such that prior loads and stores are done
+- * before, and provides an acquire ordering on success such that free()
+- * must come after.
+- *
+- * Return: true if the resulting refcount is 0, false otherwise
+- */
+-bool refcount_dec_and_test(refcount_t *r)
+-{
+- return refcount_sub_and_test(1, r);
+-}
+-EXPORT_SYMBOL(refcount_dec_and_test);
+-
+-/**
+- * refcount_dec - decrement a refcount
+- * @r: the refcount
+- *
+- * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
+- * when saturated at REFCOUNT_SATURATED.
+- *
+- * Provides release memory ordering, such that prior loads and stores are done
+- * before.
+- */
+-void refcount_dec(refcount_t *r)
+-{
+- WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
+-}
+-EXPORT_SYMBOL(refcount_dec);
+-
+-#endif /* CONFIG_REFCOUNT_FULL */
+-
+ /**
+ * refcount_dec_if_one - decrement a refcount if it is 1
+ * @r: the refcount
+--
+2.35.1
+
--- /dev/null
+From b820cde8559e0882bd6688bc9a5497bd7b2b13ec Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Thu, 21 Nov 2019 11:58:55 +0000
+Subject: locking/refcount: Remove unused refcount_*_checked() variants
+
+From: Will Deacon <will@kernel.org>
+
+[ Upstream commit 7221762c48c6bbbcc6cc51d8b803c06930215e34 ]
+
+The full-fat refcount implementation is exposed via a set of functions
+suffixed with "_checked()", the idea being that code can choose to use
+the more expensive, yet more secure implementation on a case-by-case
+basis.
+
+In reality, this hasn't happened, so with a grand total of zero users,
+let's remove the checked variants for now by simply dropping the suffix
+and predicating the out-of-line functions on CONFIG_REFCOUNT_FULL=y.
+
+Signed-off-by: Will Deacon <will@kernel.org>
+Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
+Reviewed-by: Kees Cook <keescook@chromium.org>
+Tested-by: Hanjun Guo <guohanjun@huawei.com>
+Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+Cc: Elena Reshetova <elena.reshetova@intel.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: https://lkml.kernel.org/r/20191121115902.2551-4-will@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/refcount.h | 25 ++++++-------------
+ lib/refcount.c | 54 +++++++++++++++++++++-------------------
+ 2 files changed, 36 insertions(+), 43 deletions(-)
+
+diff --git a/include/linux/refcount.h b/include/linux/refcount.h
+index 89066a1471dd..edd505d1a23b 100644
+--- a/include/linux/refcount.h
++++ b/include/linux/refcount.h
+@@ -44,32 +44,21 @@ static inline unsigned int refcount_read(const refcount_t *r)
+ return atomic_read(&r->refs);
+ }
+
+-extern __must_check bool refcount_add_not_zero_checked(int i, refcount_t *r);
+-extern void refcount_add_checked(int i, refcount_t *r);
+-
+-extern __must_check bool refcount_inc_not_zero_checked(refcount_t *r);
+-extern void refcount_inc_checked(refcount_t *r);
+-
+-extern __must_check bool refcount_sub_and_test_checked(int i, refcount_t *r);
+-
+-extern __must_check bool refcount_dec_and_test_checked(refcount_t *r);
+-extern void refcount_dec_checked(refcount_t *r);
+-
+ #ifdef CONFIG_REFCOUNT_FULL
+
+ #define REFCOUNT_MAX (UINT_MAX - 1)
+ #define REFCOUNT_SATURATED UINT_MAX
+
+-#define refcount_add_not_zero refcount_add_not_zero_checked
+-#define refcount_add refcount_add_checked
++extern __must_check bool refcount_add_not_zero(int i, refcount_t *r);
++extern void refcount_add(int i, refcount_t *r);
+
+-#define refcount_inc_not_zero refcount_inc_not_zero_checked
+-#define refcount_inc refcount_inc_checked
++extern __must_check bool refcount_inc_not_zero(refcount_t *r);
++extern void refcount_inc(refcount_t *r);
+
+-#define refcount_sub_and_test refcount_sub_and_test_checked
++extern __must_check bool refcount_sub_and_test(int i, refcount_t *r);
+
+-#define refcount_dec_and_test refcount_dec_and_test_checked
+-#define refcount_dec refcount_dec_checked
++extern __must_check bool refcount_dec_and_test(refcount_t *r);
++extern void refcount_dec(refcount_t *r);
+
+ #else
+
+diff --git a/lib/refcount.c b/lib/refcount.c
+index 719b0bc42ab1..a2f670998cee 100644
+--- a/lib/refcount.c
++++ b/lib/refcount.c
+@@ -43,8 +43,10 @@
+ #include <linux/spinlock.h>
+ #include <linux/bug.h>
+
++#ifdef CONFIG_REFCOUNT_FULL
++
+ /**
+- * refcount_add_not_zero_checked - add a value to a refcount unless it is 0
++ * refcount_add_not_zero - add a value to a refcount unless it is 0
+ * @i: the value to add to the refcount
+ * @r: the refcount
+ *
+@@ -61,7 +63,7 @@
+ *
+ * Return: false if the passed refcount is 0, true otherwise
+ */
+-bool refcount_add_not_zero_checked(int i, refcount_t *r)
++bool refcount_add_not_zero(int i, refcount_t *r)
+ {
+ unsigned int new, val = atomic_read(&r->refs);
+
+@@ -83,10 +85,10 @@ bool refcount_add_not_zero_checked(int i, refcount_t *r)
+
+ return true;
+ }
+-EXPORT_SYMBOL(refcount_add_not_zero_checked);
++EXPORT_SYMBOL(refcount_add_not_zero);
+
+ /**
+- * refcount_add_checked - add a value to a refcount
++ * refcount_add - add a value to a refcount
+ * @i: the value to add to the refcount
+ * @r: the refcount
+ *
+@@ -101,14 +103,14 @@ EXPORT_SYMBOL(refcount_add_not_zero_checked);
+ * cases, refcount_inc(), or one of its variants, should instead be used to
+ * increment a reference count.
+ */
+-void refcount_add_checked(int i, refcount_t *r)
++void refcount_add(int i, refcount_t *r)
+ {
+- WARN_ONCE(!refcount_add_not_zero_checked(i, r), "refcount_t: addition on 0; use-after-free.\n");
++ WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
+ }
+-EXPORT_SYMBOL(refcount_add_checked);
++EXPORT_SYMBOL(refcount_add);
+
+ /**
+- * refcount_inc_not_zero_checked - increment a refcount unless it is 0
++ * refcount_inc_not_zero - increment a refcount unless it is 0
+ * @r: the refcount to increment
+ *
+ * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED
+@@ -120,7 +122,7 @@ EXPORT_SYMBOL(refcount_add_checked);
+ *
+ * Return: true if the increment was successful, false otherwise
+ */
+-bool refcount_inc_not_zero_checked(refcount_t *r)
++bool refcount_inc_not_zero(refcount_t *r)
+ {
+ unsigned int new, val = atomic_read(&r->refs);
+
+@@ -140,10 +142,10 @@ bool refcount_inc_not_zero_checked(refcount_t *r)
+
+ return true;
+ }
+-EXPORT_SYMBOL(refcount_inc_not_zero_checked);
++EXPORT_SYMBOL(refcount_inc_not_zero);
+
+ /**
+- * refcount_inc_checked - increment a refcount
++ * refcount_inc - increment a refcount
+ * @r: the refcount to increment
+ *
+ * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN.
+@@ -154,14 +156,14 @@ EXPORT_SYMBOL(refcount_inc_not_zero_checked);
+ * Will WARN if the refcount is 0, as this represents a possible use-after-free
+ * condition.
+ */
+-void refcount_inc_checked(refcount_t *r)
++void refcount_inc(refcount_t *r)
+ {
+- WARN_ONCE(!refcount_inc_not_zero_checked(r), "refcount_t: increment on 0; use-after-free.\n");
++ WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
+ }
+-EXPORT_SYMBOL(refcount_inc_checked);
++EXPORT_SYMBOL(refcount_inc);
+
+ /**
+- * refcount_sub_and_test_checked - subtract from a refcount and test if it is 0
++ * refcount_sub_and_test - subtract from a refcount and test if it is 0
+ * @i: amount to subtract from the refcount
+ * @r: the refcount
+ *
+@@ -180,7 +182,7 @@ EXPORT_SYMBOL(refcount_inc_checked);
+ *
+ * Return: true if the resulting refcount is 0, false otherwise
+ */
+-bool refcount_sub_and_test_checked(int i, refcount_t *r)
++bool refcount_sub_and_test(int i, refcount_t *r)
+ {
+ unsigned int new, val = atomic_read(&r->refs);
+
+@@ -203,10 +205,10 @@ bool refcount_sub_and_test_checked(int i, refcount_t *r)
+ return false;
+
+ }
+-EXPORT_SYMBOL(refcount_sub_and_test_checked);
++EXPORT_SYMBOL(refcount_sub_and_test);
+
+ /**
+- * refcount_dec_and_test_checked - decrement a refcount and test if it is 0
++ * refcount_dec_and_test - decrement a refcount and test if it is 0
+ * @r: the refcount
+ *
+ * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
+@@ -218,14 +220,14 @@ EXPORT_SYMBOL(refcount_sub_and_test_checked);
+ *
+ * Return: true if the resulting refcount is 0, false otherwise
+ */
+-bool refcount_dec_and_test_checked(refcount_t *r)
++bool refcount_dec_and_test(refcount_t *r)
+ {
+- return refcount_sub_and_test_checked(1, r);
++ return refcount_sub_and_test(1, r);
+ }
+-EXPORT_SYMBOL(refcount_dec_and_test_checked);
++EXPORT_SYMBOL(refcount_dec_and_test);
+
+ /**
+- * refcount_dec_checked - decrement a refcount
++ * refcount_dec - decrement a refcount
+ * @r: the refcount
+ *
+ * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
+@@ -234,11 +236,13 @@ EXPORT_SYMBOL(refcount_dec_and_test_checked);
+ * Provides release memory ordering, such that prior loads and stores are done
+ * before.
+ */
+-void refcount_dec_checked(refcount_t *r)
++void refcount_dec(refcount_t *r)
+ {
+- WARN_ONCE(refcount_dec_and_test_checked(r), "refcount_t: decrement hit 0; leaking memory.\n");
++ WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
+ }
+-EXPORT_SYMBOL(refcount_dec_checked);
++EXPORT_SYMBOL(refcount_dec);
++
++#endif /* CONFIG_REFCOUNT_FULL */
+
+ /**
+ * refcount_dec_if_one - decrement a refcount if it is 1
+--
+2.35.1
+
--- /dev/null
+From fb0bf383598b6cdb897a7cb05abeb8a61963f27e Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Mon, 8 Jun 2020 21:33:14 -0700
+Subject: mmap locking API: initial implementation as rwsem wrappers
+
+From: Michel Lespinasse <walken@google.com>
+
+[ Upstream commit 9740ca4e95b43b91a4a848694a20d01ba6818f7b ]
+
+This patch series adds a new mmap locking API replacing the existing
+mmap_sem lock and unlocks. Initially the API is just implemente in terms
+of inlined rwsem calls, so it doesn't provide any new functionality.
+
+There are two justifications for the new API:
+
+- At first, it provides an easy hooking point to instrument mmap_sem
+ locking latencies independently of any other rwsems.
+
+- In the future, it may be a starting point for replacing the rwsem
+ implementation with a different one, such as range locks. This is
+ something that is being explored, even though there is no wide concensus
+ about this possible direction yet. (see
+ https://patchwork.kernel.org/cover/11401483/)
+
+This patch (of 12):
+
+This change wraps the existing mmap_sem related rwsem calls into a new
+mmap locking API. There are two justifications for the new API:
+
+- At first, it provides an easy hooking point to instrument mmap_sem
+ locking latencies independently of any other rwsems.
+
+- In the future, it may be a starting point for replacing the rwsem
+ implementation with a different one, such as range locks.
+
+Signed-off-by: Michel Lespinasse <walken@google.com>
+Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
+Reviewed-by: Daniel Jordan <daniel.m.jordan@oracle.com>
+Reviewed-by: Davidlohr Bueso <dbueso@suse.de>
+Reviewed-by: Laurent Dufour <ldufour@linux.ibm.com>
+Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Matthew Wilcox <willy@infradead.org>
+Cc: Liam Howlett <Liam.Howlett@oracle.com>
+Cc: Jerome Glisse <jglisse@redhat.com>
+Cc: David Rientjes <rientjes@google.com>
+Cc: Hugh Dickins <hughd@google.com>
+Cc: Ying Han <yinghan@google.com>
+Cc: Jason Gunthorpe <jgg@ziepe.ca>
+Cc: John Hubbard <jhubbard@nvidia.com>
+Cc: Michel Lespinasse <walken@google.com>
+Link: http://lkml.kernel.org/r/20200520052908.204642-1-walken@google.com
+Link: http://lkml.kernel.org/r/20200520052908.204642-2-walken@google.com
+Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ include/linux/mm.h | 1 +
+ include/linux/mmap_lock.h | 54 +++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 55 insertions(+)
+ create mode 100644 include/linux/mmap_lock.h
+
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index c125fea49752..d35c29d322d8 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -15,6 +15,7 @@
+ #include <linux/atomic.h>
+ #include <linux/debug_locks.h>
+ #include <linux/mm_types.h>
++#include <linux/mmap_lock.h>
+ #include <linux/range.h>
+ #include <linux/pfn.h>
+ #include <linux/percpu-refcount.h>
+diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
+new file mode 100644
+index 000000000000..97ac53b66052
+--- /dev/null
++++ b/include/linux/mmap_lock.h
+@@ -0,0 +1,54 @@
++#ifndef _LINUX_MMAP_LOCK_H
++#define _LINUX_MMAP_LOCK_H
++
++static inline void mmap_init_lock(struct mm_struct *mm)
++{
++ init_rwsem(&mm->mmap_sem);
++}
++
++static inline void mmap_write_lock(struct mm_struct *mm)
++{
++ down_write(&mm->mmap_sem);
++}
++
++static inline int mmap_write_lock_killable(struct mm_struct *mm)
++{
++ return down_write_killable(&mm->mmap_sem);
++}
++
++static inline bool mmap_write_trylock(struct mm_struct *mm)
++{
++ return down_write_trylock(&mm->mmap_sem) != 0;
++}
++
++static inline void mmap_write_unlock(struct mm_struct *mm)
++{
++ up_write(&mm->mmap_sem);
++}
++
++static inline void mmap_write_downgrade(struct mm_struct *mm)
++{
++ downgrade_write(&mm->mmap_sem);
++}
++
++static inline void mmap_read_lock(struct mm_struct *mm)
++{
++ down_read(&mm->mmap_sem);
++}
++
++static inline int mmap_read_lock_killable(struct mm_struct *mm)
++{
++ return down_read_killable(&mm->mmap_sem);
++}
++
++static inline bool mmap_read_trylock(struct mm_struct *mm)
++{
++ return down_read_trylock(&mm->mmap_sem) != 0;
++}
++
++static inline void mmap_read_unlock(struct mm_struct *mm)
++{
++ up_read(&mm->mmap_sem);
++}
++
++#endif /* _LINUX_MMAP_LOCK_H */
+--
+2.35.1
+
spi-bcm2835-bcm2835_spi_handle_err-fix-null-pointer-deref-for-non-dma-transfers.patch
mm-mempolicy-fix-uninit-value-in-mpol_rebind_policy.patch
bpf-make-sure-mac_header-was-set-before-using-it.patch
+dlm-fix-pending-remove-if-msg-allocation-fails.patch
+ima-remove-the-ima_template-kconfig-option.patch
+locking-refcount-define-constants-for-saturation-and.patch
+locking-refcount-ensure-integer-operands-are-treated.patch
+locking-refcount-remove-unused-refcount_-_checked-va.patch
+locking-refcount-move-the-bulk-of-the-refcount_full-.patch
+locking-refcount-improve-performance-of-generic-refc.patch
+locking-refcount-move-saturation-warnings-out-of-lin.patch
+locking-refcount-consolidate-refcount_-max-saturated.patch
+locking-refcount-consolidate-implementations-of-refc.patch
+x86-get-rid-of-small-constant-size-cases-in-raw_copy.patch
+x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch
+x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch-104
+mmap-locking-api-initial-implementation-as-rwsem-wra.patch
+x86-mce-deduplicate-exception-handling.patch
+bitfield.h-fix-type-of-reg-too-small-for-mask-test.patch
--- /dev/null
+From 9978eea9fcfec3bdfb3157ef9ce6b3badb31c5f6 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Sat, 15 Feb 2020 11:46:30 -0500
+Subject: x86: get rid of small constant size cases in
+ raw_copy_{to,from}_user()
+
+From: Al Viro <viro@zeniv.linux.org.uk>
+
+[ Upstream commit 4b842e4e25b12951fa10dedb4bc16bc47e3b850c ]
+
+Very few call sites where that would be triggered remain, and none
+of those is anywhere near hot enough to bother.
+
+Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ arch/x86/include/asm/uaccess.h | 12 ----
+ arch/x86/include/asm/uaccess_32.h | 27 --------
+ arch/x86/include/asm/uaccess_64.h | 108 +-----------------------------
+ 3 files changed, 2 insertions(+), 145 deletions(-)
+
+diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
+index 61d93f062a36..a19effb98fdc 100644
+--- a/arch/x86/include/asm/uaccess.h
++++ b/arch/x86/include/asm/uaccess.h
+@@ -378,18 +378,6 @@ do { \
+ : "=r" (err), ltype(x) \
+ : "m" (__m(addr)), "i" (errret), "0" (err))
+
+-#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \
+- asm volatile("\n" \
+- "1: mov"itype" %2,%"rtype"1\n" \
+- "2:\n" \
+- ".section .fixup,\"ax\"\n" \
+- "3: mov %3,%0\n" \
+- " jmp 2b\n" \
+- ".previous\n" \
+- _ASM_EXTABLE_UA(1b, 3b) \
+- : "=r" (err), ltype(x) \
+- : "m" (__m(addr)), "i" (errret), "0" (err))
+-
+ /*
+ * This doesn't do __uaccess_begin/end - the exception handling
+ * around it must do that.
+diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
+index ba2dc1930630..388a40660c7b 100644
+--- a/arch/x86/include/asm/uaccess_32.h
++++ b/arch/x86/include/asm/uaccess_32.h
+@@ -23,33 +23,6 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+ static __always_inline unsigned long
+ raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+ {
+- if (__builtin_constant_p(n)) {
+- unsigned long ret;
+-
+- switch (n) {
+- case 1:
+- ret = 0;
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u8 *)to, from, ret,
+- "b", "b", "=q", 1);
+- __uaccess_end();
+- return ret;
+- case 2:
+- ret = 0;
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u16 *)to, from, ret,
+- "w", "w", "=r", 2);
+- __uaccess_end();
+- return ret;
+- case 4:
+- ret = 0;
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u32 *)to, from, ret,
+- "l", "k", "=r", 4);
+- __uaccess_end();
+- return ret;
+- }
+- }
+ return __copy_user_ll(to, (__force const void *)from, n);
+ }
+
+diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
+index 5cd1caa8bc65..bc10e3dc64fe 100644
+--- a/arch/x86/include/asm/uaccess_64.h
++++ b/arch/x86/include/asm/uaccess_64.h
+@@ -65,117 +65,13 @@ copy_to_user_mcsafe(void *to, const void *from, unsigned len)
+ static __always_inline __must_check unsigned long
+ raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
+ {
+- int ret = 0;
+-
+- if (!__builtin_constant_p(size))
+- return copy_user_generic(dst, (__force void *)src, size);
+- switch (size) {
+- case 1:
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src,
+- ret, "b", "b", "=q", 1);
+- __uaccess_end();
+- return ret;
+- case 2:
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src,
+- ret, "w", "w", "=r", 2);
+- __uaccess_end();
+- return ret;
+- case 4:
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src,
+- ret, "l", "k", "=r", 4);
+- __uaccess_end();
+- return ret;
+- case 8:
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
+- ret, "q", "", "=r", 8);
+- __uaccess_end();
+- return ret;
+- case 10:
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
+- ret, "q", "", "=r", 10);
+- if (likely(!ret))
+- __get_user_asm_nozero(*(u16 *)(8 + (char *)dst),
+- (u16 __user *)(8 + (char __user *)src),
+- ret, "w", "w", "=r", 2);
+- __uaccess_end();
+- return ret;
+- case 16:
+- __uaccess_begin_nospec();
+- __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
+- ret, "q", "", "=r", 16);
+- if (likely(!ret))
+- __get_user_asm_nozero(*(u64 *)(8 + (char *)dst),
+- (u64 __user *)(8 + (char __user *)src),
+- ret, "q", "", "=r", 8);
+- __uaccess_end();
+- return ret;
+- default:
+- return copy_user_generic(dst, (__force void *)src, size);
+- }
++ return copy_user_generic(dst, (__force void *)src, size);
+ }
+
+ static __always_inline __must_check unsigned long
+ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
+ {
+- int ret = 0;
+-
+- if (!__builtin_constant_p(size))
+- return copy_user_generic((__force void *)dst, src, size);
+- switch (size) {
+- case 1:
+- __uaccess_begin();
+- __put_user_asm(*(u8 *)src, (u8 __user *)dst,
+- ret, "b", "b", "iq", 1);
+- __uaccess_end();
+- return ret;
+- case 2:
+- __uaccess_begin();
+- __put_user_asm(*(u16 *)src, (u16 __user *)dst,
+- ret, "w", "w", "ir", 2);
+- __uaccess_end();
+- return ret;
+- case 4:
+- __uaccess_begin();
+- __put_user_asm(*(u32 *)src, (u32 __user *)dst,
+- ret, "l", "k", "ir", 4);
+- __uaccess_end();
+- return ret;
+- case 8:
+- __uaccess_begin();
+- __put_user_asm(*(u64 *)src, (u64 __user *)dst,
+- ret, "q", "", "er", 8);
+- __uaccess_end();
+- return ret;
+- case 10:
+- __uaccess_begin();
+- __put_user_asm(*(u64 *)src, (u64 __user *)dst,
+- ret, "q", "", "er", 10);
+- if (likely(!ret)) {
+- asm("":::"memory");
+- __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
+- ret, "w", "w", "ir", 2);
+- }
+- __uaccess_end();
+- return ret;
+- case 16:
+- __uaccess_begin();
+- __put_user_asm(*(u64 *)src, (u64 __user *)dst,
+- ret, "q", "", "er", 16);
+- if (likely(!ret)) {
+- asm("":::"memory");
+- __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
+- ret, "q", "", "er", 8);
+- }
+- __uaccess_end();
+- return ret;
+- default:
+- return copy_user_generic((__force void *)dst, src, size);
+- }
++ return copy_user_generic((__force void *)dst, src, size);
+ }
+
+ static __always_inline __must_check
+--
+2.35.1
+
--- /dev/null
+From d83ef6366175fc15938a0f610f7131ba9e7a2eb3 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Wed, 8 Sep 2021 15:29:15 +0200
+Subject: x86/mce: Deduplicate exception handling
+
+From: Thomas Gleixner <tglx@linutronix.de>
+
+[ Upstream commit e42404afc4ca856c48f1e05752541faa3587c472 ]
+
+Prepare code for further simplification. No functional change.
+
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lkml.kernel.org/r/20210908132525.096452100@linutronix.de
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ arch/x86/kernel/cpu/mce/core.c | 34 +++++++++++++++++-----------------
+ 1 file changed, 17 insertions(+), 17 deletions(-)
+
+diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
+index 8a2b8e791314..9b98a7d8ac60 100644
+--- a/arch/x86/kernel/cpu/mce/core.c
++++ b/arch/x86/kernel/cpu/mce/core.c
+@@ -397,13 +397,16 @@ static int msr_to_offset(u32 msr)
+ return -1;
+ }
+
+-__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup,
+- struct pt_regs *regs, int trapnr,
+- unsigned long error_code,
+- unsigned long fault_addr)
++static void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr)
+ {
+- pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
+- (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
++ if (wrmsr) {
++ pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
++ (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
++ regs->ip, (void *)regs->ip);
++ } else {
++ pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
++ (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
++ }
+
+ show_stack_regs(regs);
+
+@@ -411,7 +414,14 @@ __visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup,
+
+ while (true)
+ cpu_relax();
++}
+
++__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup,
++ struct pt_regs *regs, int trapnr,
++ unsigned long error_code,
++ unsigned long fault_addr)
++{
++ ex_handler_msr_mce(regs, false);
+ return true;
+ }
+
+@@ -447,17 +457,7 @@ __visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup,
+ unsigned long error_code,
+ unsigned long fault_addr)
+ {
+- pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
+- (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
+- regs->ip, (void *)regs->ip);
+-
+- show_stack_regs(regs);
+-
+- panic("MCA architectural violation!\n");
+-
+- while (true)
+- cpu_relax();
+-
++ ex_handler_msr_mce(regs, true);
+ return true;
+ }
+
+--
+2.35.1
+
--- /dev/null
+From d5f813d6f5eef4f0b5e40233ef3484fb8d973a31 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Wed, 2 Feb 2022 00:49:42 +0000
+Subject: x86/uaccess: Implement macros for CMPXCHG on user addresses
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+[ Upstream commit 989b5db215a2f22f89d730b607b071d964780f10 ]
+
+Add support for CMPXCHG loops on userspace addresses. Provide both an
+"unsafe" version for tight loops that do their own uaccess begin/end, as
+well as a "safe" version for use cases where the CMPXCHG is not buried in
+a loop, e.g. KVM will resume the guest instead of looping when emulation
+of a guest atomic accesses fails the CMPXCHG.
+
+Provide 8-byte versions for 32-bit kernels so that KVM can do CMPXCHG on
+guest PAE PTEs, which are accessed via userspace addresses.
+
+Guard the asm_volatile_goto() variation with CC_HAS_ASM_GOTO_TIED_OUTPUT,
+the "+m" constraint fails on some compilers that otherwise support
+CC_HAS_ASM_GOTO_OUTPUT.
+
+Cc: stable@vger.kernel.org
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Co-developed-by: Sean Christopherson <seanjc@google.com>
+Signed-off-by: Sean Christopherson <seanjc@google.com>
+Message-Id: <20220202004945.2540433-3-seanjc@google.com>
+Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ arch/x86/include/asm/uaccess.h | 142 +++++++++++++++++++++++++++++++++
+ 1 file changed, 142 insertions(+)
+
+diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
+index a19effb98fdc..865795e2355e 100644
+--- a/arch/x86/include/asm/uaccess.h
++++ b/arch/x86/include/asm/uaccess.h
+@@ -441,6 +441,103 @@ __pu_label: \
+ __builtin_expect(__gu_err, 0); \
+ })
+
++#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
++ bool success; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm_volatile_goto("\n" \
++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
++ _ASM_EXTABLE_UA(1b, %l[label]) \
++ : CC_OUT(z) (success), \
++ [ptr] "+m" (*_ptr), \
++ [old] "+a" (__old) \
++ : [new] ltype (__new) \
++ : "memory" \
++ : label); \
++ if (unlikely(!success)) \
++ *_old = __old; \
++ likely(success); })
++
++#ifdef CONFIG_X86_32
++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
++ bool success; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm_volatile_goto("\n" \
++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
++ _ASM_EXTABLE_UA(1b, %l[label]) \
++ : CC_OUT(z) (success), \
++ "+A" (__old), \
++ [ptr] "+m" (*_ptr) \
++ : "b" ((u32)__new), \
++ "c" ((u32)((u64)__new >> 32)) \
++ : "memory" \
++ : label); \
++ if (unlikely(!success)) \
++ *_old = __old; \
++ likely(success); })
++#endif // CONFIG_X86_32
++#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
++ int __err = 0; \
++ bool success; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm volatile("\n" \
++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
++ CC_SET(z) \
++ "2:\n" \
++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \
++ %[errout]) \
++ : CC_OUT(z) (success), \
++ [errout] "+r" (__err), \
++ [ptr] "+m" (*_ptr), \
++ [old] "+a" (__old) \
++ : [new] ltype (__new) \
++ : "memory", "cc"); \
++ if (unlikely(__err)) \
++ goto label; \
++ if (unlikely(!success)) \
++ *_old = __old; \
++ likely(success); })
++
++#ifdef CONFIG_X86_32
++/*
++ * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error.
++ * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
++ * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses
++ * both ESI and EDI for the memory operand, compilation will fail if the error
++ * is an input+output as there will be no register available for input.
++ */
++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
++ int __result; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm volatile("\n" \
++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
++ "mov $0, %%ecx\n\t" \
++ "setz %%cl\n" \
++ "2:\n" \
++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \
++ : [result]"=c" (__result), \
++ "+A" (__old), \
++ [ptr] "+m" (*_ptr) \
++ : "b" ((u32)__new), \
++ "c" ((u32)((u64)__new >> 32)) \
++ : "memory", "cc"); \
++ if (unlikely(__result < 0)) \
++ goto label; \
++ if (unlikely(!__result)) \
++ *_old = __old; \
++ likely(__result); })
++#endif // CONFIG_X86_32
++#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
++
+ /* FIXME: this hack is definitely wrong -AK */
+ struct __large_struct { unsigned long buf[100]; };
+ #define __m(x) (*(struct __large_struct __user *)(x))
+@@ -722,6 +819,51 @@ do { \
+ if (unlikely(__gu_err)) goto err_label; \
+ } while (0)
+
++extern void __try_cmpxchg_user_wrong_size(void);
++
++#ifndef CONFIG_X86_32
++#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \
++ __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
++#endif
++
++/*
++ * Force the pointer to u<size> to match the size expected by the asm helper.
++ * clang/LLVM compiles all cases and only discards the unused paths after
++ * processing errors, which breaks i386 if the pointer is an 8-byte value.
++ */
++#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
++ bool __ret; \
++ __chk_user_ptr(_ptr); \
++ switch (sizeof(*(_ptr))) { \
++ case 1: __ret = __try_cmpxchg_user_asm("b", "q", \
++ (__force u8 *)(_ptr), (_oldp), \
++ (_nval), _label); \
++ break; \
++ case 2: __ret = __try_cmpxchg_user_asm("w", "r", \
++ (__force u16 *)(_ptr), (_oldp), \
++ (_nval), _label); \
++ break; \
++ case 4: __ret = __try_cmpxchg_user_asm("l", "r", \
++ (__force u32 *)(_ptr), (_oldp), \
++ (_nval), _label); \
++ break; \
++ case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
++ (_nval), _label); \
++ break; \
++ default: __try_cmpxchg_user_wrong_size(); \
++ } \
++ __ret; })
++
++/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
++#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
++ int __ret = -EFAULT; \
++ __uaccess_begin_nospec(); \
++ __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \
++_label: \
++ __uaccess_end(); \
++ __ret; \
++ })
++
+ /*
+ * We want the unsafe accessors to always be inlined and use
+ * the error labels - thus the macro games.
+--
+2.35.1
+
--- /dev/null
+From b64be47e3d6b941b1934109ad7388ef0748547fc Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Wed, 2 Feb 2022 00:49:42 +0000
+Subject: x86/uaccess: Implement macros for CMPXCHG on user addresses
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+[ Upstream commit 989b5db215a2f22f89d730b607b071d964780f10 ]
+
+Add support for CMPXCHG loops on userspace addresses. Provide both an
+"unsafe" version for tight loops that do their own uaccess begin/end, as
+well as a "safe" version for use cases where the CMPXCHG is not buried in
+a loop, e.g. KVM will resume the guest instead of looping when emulation
+of a guest atomic accesses fails the CMPXCHG.
+
+Provide 8-byte versions for 32-bit kernels so that KVM can do CMPXCHG on
+guest PAE PTEs, which are accessed via userspace addresses.
+
+Guard the asm_volatile_goto() variation with CC_HAS_ASM_GOTO_TIED_OUTPUT,
+the "+m" constraint fails on some compilers that otherwise support
+CC_HAS_ASM_GOTO_OUTPUT.
+
+Cc: stable@vger.kernel.org
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Co-developed-by: Sean Christopherson <seanjc@google.com>
+Signed-off-by: Sean Christopherson <seanjc@google.com>
+Message-Id: <20220202004945.2540433-3-seanjc@google.com>
+Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ arch/x86/include/asm/uaccess.h | 142 +++++++++++++++++++++++++++++++++
+ 1 file changed, 142 insertions(+)
+
+diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
+index 865795e2355e..7299acbbb7a6 100644
+--- a/arch/x86/include/asm/uaccess.h
++++ b/arch/x86/include/asm/uaccess.h
+@@ -538,6 +538,103 @@ __pu_label: \
+ #endif // CONFIG_X86_32
+ #endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
+
++#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
++ bool success; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm_volatile_goto("\n" \
++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
++ _ASM_EXTABLE_UA(1b, %l[label]) \
++ : CC_OUT(z) (success), \
++ [ptr] "+m" (*_ptr), \
++ [old] "+a" (__old) \
++ : [new] ltype (__new) \
++ : "memory" \
++ : label); \
++ if (unlikely(!success)) \
++ *_old = __old; \
++ likely(success); })
++
++#ifdef CONFIG_X86_32
++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
++ bool success; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm_volatile_goto("\n" \
++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
++ _ASM_EXTABLE_UA(1b, %l[label]) \
++ : CC_OUT(z) (success), \
++ "+A" (__old), \
++ [ptr] "+m" (*_ptr) \
++ : "b" ((u32)__new), \
++ "c" ((u32)((u64)__new >> 32)) \
++ : "memory" \
++ : label); \
++ if (unlikely(!success)) \
++ *_old = __old; \
++ likely(success); })
++#endif // CONFIG_X86_32
++#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
++ int __err = 0; \
++ bool success; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm volatile("\n" \
++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
++ CC_SET(z) \
++ "2:\n" \
++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \
++ %[errout]) \
++ : CC_OUT(z) (success), \
++ [errout] "+r" (__err), \
++ [ptr] "+m" (*_ptr), \
++ [old] "+a" (__old) \
++ : [new] ltype (__new) \
++ : "memory", "cc"); \
++ if (unlikely(__err)) \
++ goto label; \
++ if (unlikely(!success)) \
++ *_old = __old; \
++ likely(success); })
++
++#ifdef CONFIG_X86_32
++/*
++ * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error.
++ * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
++ * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses
++ * both ESI and EDI for the memory operand, compilation will fail if the error
++ * is an input+output as there will be no register available for input.
++ */
++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
++ int __result; \
++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
++ __typeof__(*(_ptr)) __old = *_old; \
++ __typeof__(*(_ptr)) __new = (_new); \
++ asm volatile("\n" \
++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
++ "mov $0, %%ecx\n\t" \
++ "setz %%cl\n" \
++ "2:\n" \
++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \
++ : [result]"=c" (__result), \
++ "+A" (__old), \
++ [ptr] "+m" (*_ptr) \
++ : "b" ((u32)__new), \
++ "c" ((u32)((u64)__new >> 32)) \
++ : "memory", "cc"); \
++ if (unlikely(__result < 0)) \
++ goto label; \
++ if (unlikely(!__result)) \
++ *_old = __old; \
++ likely(__result); })
++#endif // CONFIG_X86_32
++#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
++
+ /* FIXME: this hack is definitely wrong -AK */
+ struct __large_struct { unsigned long buf[100]; };
+ #define __m(x) (*(struct __large_struct __user *)(x))
+@@ -826,6 +923,51 @@ extern void __try_cmpxchg_user_wrong_size(void);
+ __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
+ #endif
+
++/*
++ * Force the pointer to u<size> to match the size expected by the asm helper.
++ * clang/LLVM compiles all cases and only discards the unused paths after
++ * processing errors, which breaks i386 if the pointer is an 8-byte value.
++ */
++#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
++ bool __ret; \
++ __chk_user_ptr(_ptr); \
++ switch (sizeof(*(_ptr))) { \
++ case 1: __ret = __try_cmpxchg_user_asm("b", "q", \
++ (__force u8 *)(_ptr), (_oldp), \
++ (_nval), _label); \
++ break; \
++ case 2: __ret = __try_cmpxchg_user_asm("w", "r", \
++ (__force u16 *)(_ptr), (_oldp), \
++ (_nval), _label); \
++ break; \
++ case 4: __ret = __try_cmpxchg_user_asm("l", "r", \
++ (__force u32 *)(_ptr), (_oldp), \
++ (_nval), _label); \
++ break; \
++ case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
++ (_nval), _label); \
++ break; \
++ default: __try_cmpxchg_user_wrong_size(); \
++ } \
++ __ret; })
++
++/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
++#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
++ int __ret = -EFAULT; \
++ __uaccess_begin_nospec(); \
++ __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \
++_label: \
++ __uaccess_end(); \
++ __ret; \
++ })
++
++extern void __try_cmpxchg_user_wrong_size(void);
++
++#ifndef CONFIG_X86_32
++#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \
++ __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
++#endif
++
+ /*
+ * Force the pointer to u<size> to match the size expected by the asm helper.
+ * clang/LLVM compiles all cases and only discards the unused paths after
+--
+2.35.1
+
projects / thirdparty / kernel / stable-queue.git / commitdiff
