From: Sasha Levin Date: Mon, 25 Jul 2022 18:15:16 +0000 (-0400) Subject: Fixes for 5.4 X-Git-Tag: v4.9.325~42 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=2d92f6eaf14f73e9dc50eb95dd9ae923bd521088;p=thirdparty%2Fkernel%2Fstable-queue.git Fixes for 5.4 Signed-off-by: Sasha Levin --- diff --git a/queue-5.4/bitfield.h-fix-type-of-reg-too-small-for-mask-test.patch b/queue-5.4/bitfield.h-fix-type-of-reg-too-small-for-mask-test.patch new file mode 100644 index 00000000000..72f38f82833 --- /dev/null +++ b/queue-5.4/bitfield.h-fix-type-of-reg-too-small-for-mask-test.patch @@ -0,0 +1,63 @@ +From 7df8ac8777bc608ca41b7f9defd14bf710bdca7b Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Wed, 10 Nov 2021 11:01:03 +0100 +Subject: bitfield.h: Fix "type of reg too small for mask" test + +From: Peter Zijlstra + +[ Upstream commit bff8c3848e071d387d8b0784dc91fa49cd563774 ] + +The test: 'mask > (typeof(_reg))~0ull' only works correctly when both +sides are unsigned, consider: + + - 0xff000000 vs (int)~0ull + - 0x000000ff vs (int)~0ull + +Signed-off-by: Peter Zijlstra (Intel) +Reviewed-by: Josh Poimboeuf +Link: https://lore.kernel.org/r/20211110101324.950210584@infradead.org +Signed-off-by: Sasha Levin +--- + include/linux/bitfield.h | 19 ++++++++++++++++++- + 1 file changed, 18 insertions(+), 1 deletion(-) + +diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h +index 4c0224ff0a14..4f1c0f8e1bb0 100644 +--- a/include/linux/bitfield.h ++++ b/include/linux/bitfield.h +@@ -41,6 +41,22 @@ + + #define __bf_shf(x) (__builtin_ffsll(x) - 1) + ++#define __scalar_type_to_unsigned_cases(type) \ ++ unsigned type: (unsigned type)0, \ ++ signed type: (unsigned type)0 ++ ++#define __unsigned_scalar_typeof(x) typeof( \ ++ _Generic((x), \ ++ char: (unsigned char)0, \ ++ __scalar_type_to_unsigned_cases(char), \ ++ __scalar_type_to_unsigned_cases(short), \ ++ __scalar_type_to_unsigned_cases(int), \ ++ __scalar_type_to_unsigned_cases(long), \ ++ __scalar_type_to_unsigned_cases(long long), \ ++ default: (x))) ++ ++#define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x)) ++ + #define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \ + ({ \ + BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \ +@@ -49,7 +65,8 @@ + BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \ + ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \ + _pfx "value too large for the field"); \ +- BUILD_BUG_ON_MSG((_mask) > (typeof(_reg))~0ull, \ ++ BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \ ++ __bf_cast_unsigned(_reg, ~0ull), \ + _pfx "type of reg too small for mask"); \ + __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \ + (1ULL << __bf_shf(_mask))); \ +-- +2.35.1 + diff --git a/queue-5.4/dlm-fix-pending-remove-if-msg-allocation-fails.patch b/queue-5.4/dlm-fix-pending-remove-if-msg-allocation-fails.patch new file mode 100644 index 00000000000..201d26093ba --- /dev/null +++ b/queue-5.4/dlm-fix-pending-remove-if-msg-allocation-fails.patch @@ -0,0 +1,47 @@ +From 2445f6197707f527534b5277ebc0689c6211174f Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Wed, 6 Apr 2022 13:34:16 -0400 +Subject: dlm: fix pending remove if msg allocation fails + +From: Alexander Aring + +[ Upstream commit ba58995909b5098ca4003af65b0ccd5a8d13dd25 ] + +This patch unsets ls_remove_len and ls_remove_name if a message +allocation of a remove messages fails. In this case we never send a +remove message out but set the per ls ls_remove_len ls_remove_name +variable for a pending remove. Unset those variable should indicate +possible waiters in wait_pending_remove() that no pending remove is +going on at this moment. + +Cc: stable@vger.kernel.org +Signed-off-by: Alexander Aring +Signed-off-by: David Teigland +Signed-off-by: Sasha Levin +--- + fs/dlm/lock.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c +index 4ae8becdb51d..9165bf56c6e8 100644 +--- a/fs/dlm/lock.c ++++ b/fs/dlm/lock.c +@@ -4067,13 +4067,14 @@ static void send_repeat_remove(struct dlm_ls *ls, char *ms_name, int len) + rv = _create_message(ls, sizeof(struct dlm_message) + len, + dir_nodeid, DLM_MSG_REMOVE, &ms, &mh); + if (rv) +- return; ++ goto out; + + memcpy(ms->m_extra, name, len); + ms->m_hash = hash; + + send_message(mh, ms); + ++out: + spin_lock(&ls->ls_remove_spin); + ls->ls_remove_len = 0; + memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN); +-- +2.35.1 + diff --git a/queue-5.4/ima-remove-the-ima_template-kconfig-option.patch b/queue-5.4/ima-remove-the-ima_template-kconfig-option.patch new file mode 100644 index 00000000000..58a2942137a --- /dev/null +++ b/queue-5.4/ima-remove-the-ima_template-kconfig-option.patch @@ -0,0 +1,100 @@ +From 96bd7f547174e7069b0cd92f4d30e363b5f06e23 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 7 Apr 2022 10:16:19 +0800 +Subject: ima: remove the IMA_TEMPLATE Kconfig option + +From: GUO Zihua + +[ Upstream commit 891163adf180bc369b2f11c9dfce6d2758d2a5bd ] + +The original 'ima' measurement list template contains a hash, defined +as 20 bytes, and a null terminated pathname, limited to 255 +characters. Other measurement list templates permit both larger hashes +and longer pathnames. When the "ima" template is configured as the +default, a new measurement list template (ima_template=) must be +specified before specifying a larger hash algorithm (ima_hash=) on the +boot command line. + +To avoid this boot command line ordering issue, remove the legacy "ima" +template configuration option, allowing it to still be specified on the +boot command line. + +The root cause of this issue is that during the processing of ima_hash, +we would try to check whether the hash algorithm is compatible with the +template. If the template is not set at the moment we do the check, we +check the algorithm against the configured default template. If the +default template is "ima", then we reject any hash algorithm other than +sha1 and md5. + +For example, if the compiled default template is "ima", and the default +algorithm is sha1 (which is the current default). In the cmdline, we put +in "ima_hash=sha256 ima_template=ima-ng". The expected behavior would be +that ima starts with ima-ng as the template and sha256 as the hash +algorithm. However, during the processing of "ima_hash=", +"ima_template=" has not been processed yet, and hash_setup would check +the configured hash algorithm against the compiled default: ima, and +reject sha256. So at the end, the hash algorithm that is actually used +will be sha1. + +With template "ima" removed from the configured default, we ensure that +the default tempalte would at least be "ima-ng" which allows for +basically any hash algorithm. + +This change would not break the algorithm compatibility checks for IMA. + +Fixes: 4286587dccd43 ("ima: add Kconfig default measurement list template") +Signed-off-by: GUO Zihua +Cc: +Signed-off-by: Mimi Zohar +Signed-off-by: Sasha Levin +--- + security/integrity/ima/Kconfig | 12 +++++------- + 1 file changed, 5 insertions(+), 7 deletions(-) + +diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig +index 748f3ee27b23..44b3315f3235 100644 +--- a/security/integrity/ima/Kconfig ++++ b/security/integrity/ima/Kconfig +@@ -69,10 +69,9 @@ choice + hash, defined as 20 bytes, and a null terminated pathname, + limited to 255 characters. The 'ima-ng' measurement list + template permits both larger hash digests and longer +- pathnames. ++ pathnames. The configured default template can be replaced ++ by specifying "ima_template=" on the boot command line. + +- config IMA_TEMPLATE +- bool "ima" + config IMA_NG_TEMPLATE + bool "ima-ng (default)" + config IMA_SIG_TEMPLATE +@@ -82,7 +81,6 @@ endchoice + config IMA_DEFAULT_TEMPLATE + string + depends on IMA +- default "ima" if IMA_TEMPLATE + default "ima-ng" if IMA_NG_TEMPLATE + default "ima-sig" if IMA_SIG_TEMPLATE + +@@ -102,15 +100,15 @@ choice + + config IMA_DEFAULT_HASH_SHA256 + bool "SHA256" +- depends on CRYPTO_SHA256=y && !IMA_TEMPLATE ++ depends on CRYPTO_SHA256=y + + config IMA_DEFAULT_HASH_SHA512 + bool "SHA512" +- depends on CRYPTO_SHA512=y && !IMA_TEMPLATE ++ depends on CRYPTO_SHA512=y + + config IMA_DEFAULT_HASH_WP512 + bool "WP512" +- depends on CRYPTO_WP512=y && !IMA_TEMPLATE ++ depends on CRYPTO_WP512=y + endchoice + + config IMA_DEFAULT_HASH +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-consolidate-implementations-of-refc.patch b/queue-5.4/locking-refcount-consolidate-implementations-of-refc.patch new file mode 100644 index 00000000000..730e737b58d --- /dev/null +++ b/queue-5.4/locking-refcount-consolidate-implementations-of-refc.patch @@ -0,0 +1,544 @@ +From 4afe87339436d8e4a820c4ad24a9e3880e45a6c4 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:59:00 +0000 +Subject: locking/refcount: Consolidate implementations of refcount_t + +From: Will Deacon + +[ Upstream commit fb041bb7c0a918b95c6889fc965cdc4a75b4c0ca ] + +The generic implementation of refcount_t should be good enough for +everybody, so remove ARCH_HAS_REFCOUNT and REFCOUNT_FULL entirely, +leaving the generic implementation enabled unconditionally. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Acked-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-9-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + arch/Kconfig | 21 ---- + arch/arm/Kconfig | 1 - + arch/arm64/Kconfig | 1 - + arch/s390/configs/debug_defconfig | 1 - + arch/x86/Kconfig | 1 - + arch/x86/include/asm/asm.h | 6 -- + arch/x86/include/asm/refcount.h | 126 ----------------------- + arch/x86/mm/extable.c | 49 --------- + drivers/gpu/drm/i915/Kconfig.debug | 1 - + include/linux/refcount.h | 158 +++++++++++------------------ + lib/refcount.c | 2 +- + 11 files changed, 59 insertions(+), 308 deletions(-) + delete mode 100644 arch/x86/include/asm/refcount.h + +diff --git a/arch/Kconfig b/arch/Kconfig +index a8df66e64544..2219a07dca1e 100644 +--- a/arch/Kconfig ++++ b/arch/Kconfig +@@ -915,27 +915,6 @@ config STRICT_MODULE_RWX + config ARCH_HAS_PHYS_TO_DMA + bool + +-config ARCH_HAS_REFCOUNT +- bool +- help +- An architecture selects this when it has implemented refcount_t +- using open coded assembly primitives that provide an optimized +- refcount_t implementation, possibly at the expense of some full +- refcount state checks of CONFIG_REFCOUNT_FULL=y. +- +- The refcount overflow check behavior, however, must be retained. +- Catching overflows is the primary security concern for protecting +- against bugs in reference counts. +- +-config REFCOUNT_FULL +- bool "Perform full reference count validation at the expense of speed" +- help +- Enabling this switches the refcounting infrastructure from a fast +- unchecked atomic_t implementation to a fully state checked +- implementation, which can be (slightly) slower but provides protections +- against various use-after-free conditions that can be used in +- security flaw exploits. +- + config HAVE_ARCH_COMPILER_H + bool + help +diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig +index a1622b9290fd..a4364cce85f8 100644 +--- a/arch/arm/Kconfig ++++ b/arch/arm/Kconfig +@@ -119,7 +119,6 @@ config ARM + select OLD_SIGSUSPEND3 + select PCI_SYSCALL if PCI + select PERF_USE_VMALLOC +- select REFCOUNT_FULL + select RTC_LIB + select SYS_SUPPORTS_APM_EMULATION + # Above selects are sorted alphabetically; please add new ones +diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig +index a1a828ca188c..6b73143f0cf8 100644 +--- a/arch/arm64/Kconfig ++++ b/arch/arm64/Kconfig +@@ -181,7 +181,6 @@ config ARM64 + select PCI_SYSCALL if PCI + select POWER_RESET + select POWER_SUPPLY +- select REFCOUNT_FULL + select SPARSE_IRQ + select SWIOTLB + select SYSCTL_EXCEPTION_TRACE +diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig +index 38d64030aacf..2e60c80395ab 100644 +--- a/arch/s390/configs/debug_defconfig ++++ b/arch/s390/configs/debug_defconfig +@@ -62,7 +62,6 @@ CONFIG_OPROFILE=m + CONFIG_KPROBES=y + CONFIG_JUMP_LABEL=y + CONFIG_STATIC_KEYS_SELFTEST=y +-CONFIG_REFCOUNT_FULL=y + CONFIG_LOCK_EVENT_COUNTS=y + CONFIG_MODULES=y + CONFIG_MODULE_FORCE_LOAD=y +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index c6c71592f6e4..6002252692af 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -73,7 +73,6 @@ config X86 + select ARCH_HAS_PMEM_API if X86_64 + select ARCH_HAS_PTE_DEVMAP if X86_64 + select ARCH_HAS_PTE_SPECIAL +- select ARCH_HAS_REFCOUNT + select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 + select ARCH_HAS_UACCESS_MCSAFE if X86_64 && X86_MCE + select ARCH_HAS_SET_MEMORY +diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h +index 1b563f9167ea..cd339b88d5d4 100644 +--- a/arch/x86/include/asm/asm.h ++++ b/arch/x86/include/asm/asm.h +@@ -141,9 +141,6 @@ + # define _ASM_EXTABLE_EX(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext) + +-# define _ASM_EXTABLE_REFCOUNT(from, to) \ +- _ASM_EXTABLE_HANDLE(from, to, ex_handler_refcount) +- + # define _ASM_NOKPROBE(entry) \ + .pushsection "_kprobe_blacklist","aw" ; \ + _ASM_ALIGN ; \ +@@ -172,9 +169,6 @@ + # define _ASM_EXTABLE_EX(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext) + +-# define _ASM_EXTABLE_REFCOUNT(from, to) \ +- _ASM_EXTABLE_HANDLE(from, to, ex_handler_refcount) +- + /* For C file, we already have NOKPROBE_SYMBOL macro */ + #endif + +diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h +deleted file mode 100644 +index 232f856e0db0..000000000000 +--- a/arch/x86/include/asm/refcount.h ++++ /dev/null +@@ -1,126 +0,0 @@ +-#ifndef __ASM_X86_REFCOUNT_H +-#define __ASM_X86_REFCOUNT_H +-/* +- * x86-specific implementation of refcount_t. Based on PAX_REFCOUNT from +- * PaX/grsecurity. +- */ +-#include +-#include +- +-/* +- * This is the first portion of the refcount error handling, which lives in +- * .text.unlikely, and is jumped to from the CPU flag check (in the +- * following macros). This saves the refcount value location into CX for +- * the exception handler to use (in mm/extable.c), and then triggers the +- * central refcount exception. The fixup address for the exception points +- * back to the regular execution flow in .text. +- */ +-#define _REFCOUNT_EXCEPTION \ +- ".pushsection .text..refcount\n" \ +- "111:\tlea %[var], %%" _ASM_CX "\n" \ +- "112:\t" ASM_UD2 "\n" \ +- ASM_UNREACHABLE \ +- ".popsection\n" \ +- "113:\n" \ +- _ASM_EXTABLE_REFCOUNT(112b, 113b) +- +-/* Trigger refcount exception if refcount result is negative. */ +-#define REFCOUNT_CHECK_LT_ZERO \ +- "js 111f\n\t" \ +- _REFCOUNT_EXCEPTION +- +-/* Trigger refcount exception if refcount result is zero or negative. */ +-#define REFCOUNT_CHECK_LE_ZERO \ +- "jz 111f\n\t" \ +- REFCOUNT_CHECK_LT_ZERO +- +-/* Trigger refcount exception unconditionally. */ +-#define REFCOUNT_ERROR \ +- "jmp 111f\n\t" \ +- _REFCOUNT_EXCEPTION +- +-static __always_inline void refcount_add(unsigned int i, refcount_t *r) +-{ +- asm volatile(LOCK_PREFIX "addl %1,%0\n\t" +- REFCOUNT_CHECK_LT_ZERO +- : [var] "+m" (r->refs.counter) +- : "ir" (i) +- : "cc", "cx"); +-} +- +-static __always_inline void refcount_inc(refcount_t *r) +-{ +- asm volatile(LOCK_PREFIX "incl %0\n\t" +- REFCOUNT_CHECK_LT_ZERO +- : [var] "+m" (r->refs.counter) +- : : "cc", "cx"); +-} +- +-static __always_inline void refcount_dec(refcount_t *r) +-{ +- asm volatile(LOCK_PREFIX "decl %0\n\t" +- REFCOUNT_CHECK_LE_ZERO +- : [var] "+m" (r->refs.counter) +- : : "cc", "cx"); +-} +- +-static __always_inline __must_check +-bool refcount_sub_and_test(unsigned int i, refcount_t *r) +-{ +- bool ret = GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl", +- REFCOUNT_CHECK_LT_ZERO, +- r->refs.counter, e, "er", i, "cx"); +- +- if (ret) { +- smp_acquire__after_ctrl_dep(); +- return true; +- } +- +- return false; +-} +- +-static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r) +-{ +- bool ret = GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl", +- REFCOUNT_CHECK_LT_ZERO, +- r->refs.counter, e, "cx"); +- +- if (ret) { +- smp_acquire__after_ctrl_dep(); +- return true; +- } +- +- return false; +-} +- +-static __always_inline __must_check +-bool refcount_add_not_zero(unsigned int i, refcount_t *r) +-{ +- int c, result; +- +- c = atomic_read(&(r->refs)); +- do { +- if (unlikely(c == 0)) +- return false; +- +- result = c + i; +- +- /* Did we try to increment from/to an undesirable state? */ +- if (unlikely(c < 0 || c == INT_MAX || result < c)) { +- asm volatile(REFCOUNT_ERROR +- : : [var] "m" (r->refs.counter) +- : "cc", "cx"); +- break; +- } +- +- } while (!atomic_try_cmpxchg(&(r->refs), &c, result)); +- +- return c != 0; +-} +- +-static __always_inline __must_check bool refcount_inc_not_zero(refcount_t *r) +-{ +- return refcount_add_not_zero(1, r); +-} +- +-#endif +diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c +index 4d75bc656f97..30bb0bd3b1b8 100644 +--- a/arch/x86/mm/extable.c ++++ b/arch/x86/mm/extable.c +@@ -44,55 +44,6 @@ __visible bool ex_handler_fault(const struct exception_table_entry *fixup, + } + EXPORT_SYMBOL_GPL(ex_handler_fault); + +-/* +- * Handler for UD0 exception following a failed test against the +- * result of a refcount inc/dec/add/sub. +- */ +-__visible bool ex_handler_refcount(const struct exception_table_entry *fixup, +- struct pt_regs *regs, int trapnr, +- unsigned long error_code, +- unsigned long fault_addr) +-{ +- /* First unconditionally saturate the refcount. */ +- *(int *)regs->cx = INT_MIN / 2; +- +- /* +- * Strictly speaking, this reports the fixup destination, not +- * the fault location, and not the actually overflowing +- * instruction, which is the instruction before the "js", but +- * since that instruction could be a variety of lengths, just +- * report the location after the overflow, which should be close +- * enough for finding the overflow, as it's at least back in +- * the function, having returned from .text.unlikely. +- */ +- regs->ip = ex_fixup_addr(fixup); +- +- /* +- * This function has been called because either a negative refcount +- * value was seen by any of the refcount functions, or a zero +- * refcount value was seen by refcount_dec(). +- * +- * If we crossed from INT_MAX to INT_MIN, OF (Overflow Flag: result +- * wrapped around) will be set. Additionally, seeing the refcount +- * reach 0 will set ZF (Zero Flag: result was zero). In each of +- * these cases we want a report, since it's a boundary condition. +- * The SF case is not reported since it indicates post-boundary +- * manipulations below zero or above INT_MAX. And if none of the +- * flags are set, something has gone very wrong, so report it. +- */ +- if (regs->flags & (X86_EFLAGS_OF | X86_EFLAGS_ZF)) { +- bool zero = regs->flags & X86_EFLAGS_ZF; +- +- refcount_error_report(regs, zero ? "hit zero" : "overflow"); +- } else if ((regs->flags & X86_EFLAGS_SF) == 0) { +- /* Report if none of OF, ZF, nor SF are set. */ +- refcount_error_report(regs, "unexpected saturation"); +- } +- +- return true; +-} +-EXPORT_SYMBOL(ex_handler_refcount); +- + /* + * Handler for when we fail to restore a task's FPU state. We should never get + * here because the FPU state of a task using the FPU (task->thread.fpu.state) +diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug +index 41c8e39a73ba..e4f03fcb125e 100644 +--- a/drivers/gpu/drm/i915/Kconfig.debug ++++ b/drivers/gpu/drm/i915/Kconfig.debug +@@ -21,7 +21,6 @@ config DRM_I915_DEBUG + depends on DRM_I915 + select DEBUG_FS + select PREEMPT_COUNT +- select REFCOUNT_FULL + select I2C_CHARDEV + select STACKDEPOT + select DRM_DP_AUX_CHARDEV +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index 757d4630115c..0ac50cf62d06 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -1,64 +1,4 @@ + /* SPDX-License-Identifier: GPL-2.0 */ +-#ifndef _LINUX_REFCOUNT_H +-#define _LINUX_REFCOUNT_H +- +-#include +-#include +-#include +-#include +- +-struct mutex; +- +-/** +- * struct refcount_t - variant of atomic_t specialized for reference counts +- * @refs: atomic_t counter field +- * +- * The counter saturates at REFCOUNT_SATURATED and will not move once +- * there. This avoids wrapping the counter and causing 'spurious' +- * use-after-free bugs. +- */ +-typedef struct refcount_struct { +- atomic_t refs; +-} refcount_t; +- +-#define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), } +-#define REFCOUNT_MAX INT_MAX +-#define REFCOUNT_SATURATED (INT_MIN / 2) +- +-enum refcount_saturation_type { +- REFCOUNT_ADD_NOT_ZERO_OVF, +- REFCOUNT_ADD_OVF, +- REFCOUNT_ADD_UAF, +- REFCOUNT_SUB_UAF, +- REFCOUNT_DEC_LEAK, +-}; +- +-void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t); +- +-/** +- * refcount_set - set a refcount's value +- * @r: the refcount +- * @n: value to which the refcount will be set +- */ +-static inline void refcount_set(refcount_t *r, int n) +-{ +- atomic_set(&r->refs, n); +-} +- +-/** +- * refcount_read - get a refcount's value +- * @r: the refcount +- * +- * Return: the refcount's value +- */ +-static inline unsigned int refcount_read(const refcount_t *r) +-{ +- return atomic_read(&r->refs); +-} +- +-#ifdef CONFIG_REFCOUNT_FULL +-#include +- + /* + * Variant of atomic_t specialized for reference counts. + * +@@ -136,6 +76,64 @@ static inline unsigned int refcount_read(const refcount_t *r) + * + */ + ++#ifndef _LINUX_REFCOUNT_H ++#define _LINUX_REFCOUNT_H ++ ++#include ++#include ++#include ++#include ++#include ++ ++struct mutex; ++ ++/** ++ * struct refcount_t - variant of atomic_t specialized for reference counts ++ * @refs: atomic_t counter field ++ * ++ * The counter saturates at REFCOUNT_SATURATED and will not move once ++ * there. This avoids wrapping the counter and causing 'spurious' ++ * use-after-free bugs. ++ */ ++typedef struct refcount_struct { ++ atomic_t refs; ++} refcount_t; ++ ++#define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), } ++#define REFCOUNT_MAX INT_MAX ++#define REFCOUNT_SATURATED (INT_MIN / 2) ++ ++enum refcount_saturation_type { ++ REFCOUNT_ADD_NOT_ZERO_OVF, ++ REFCOUNT_ADD_OVF, ++ REFCOUNT_ADD_UAF, ++ REFCOUNT_SUB_UAF, ++ REFCOUNT_DEC_LEAK, ++}; ++ ++void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t); ++ ++/** ++ * refcount_set - set a refcount's value ++ * @r: the refcount ++ * @n: value to which the refcount will be set ++ */ ++static inline void refcount_set(refcount_t *r, int n) ++{ ++ atomic_set(&r->refs, n); ++} ++ ++/** ++ * refcount_read - get a refcount's value ++ * @r: the refcount ++ * ++ * Return: the refcount's value ++ */ ++static inline unsigned int refcount_read(const refcount_t *r) ++{ ++ return atomic_read(&r->refs); ++} ++ + /** + * refcount_add_not_zero - add a value to a refcount unless it is 0 + * @i: the value to add to the refcount +@@ -298,46 +296,6 @@ static inline void refcount_dec(refcount_t *r) + if (unlikely(atomic_fetch_sub_release(1, &r->refs) <= 1)) + refcount_warn_saturate(r, REFCOUNT_DEC_LEAK); + } +-#else /* CONFIG_REFCOUNT_FULL */ +-# ifdef CONFIG_ARCH_HAS_REFCOUNT +-# include +-# else +-static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) +-{ +- return atomic_add_unless(&r->refs, i, 0); +-} +- +-static inline void refcount_add(int i, refcount_t *r) +-{ +- atomic_add(i, &r->refs); +-} +- +-static inline __must_check bool refcount_inc_not_zero(refcount_t *r) +-{ +- return atomic_add_unless(&r->refs, 1, 0); +-} +- +-static inline void refcount_inc(refcount_t *r) +-{ +- atomic_inc(&r->refs); +-} +- +-static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) +-{ +- return atomic_sub_and_test(i, &r->refs); +-} +- +-static inline __must_check bool refcount_dec_and_test(refcount_t *r) +-{ +- return atomic_dec_and_test(&r->refs); +-} +- +-static inline void refcount_dec(refcount_t *r) +-{ +- atomic_dec(&r->refs); +-} +-# endif /* !CONFIG_ARCH_HAS_REFCOUNT */ +-#endif /* !CONFIG_REFCOUNT_FULL */ + + extern __must_check bool refcount_dec_if_one(refcount_t *r); + extern __must_check bool refcount_dec_not_one(refcount_t *r); +diff --git a/lib/refcount.c b/lib/refcount.c +index 8b7e249c0e10..ebac8b7d15a7 100644 +--- a/lib/refcount.c ++++ b/lib/refcount.c +@@ -1,6 +1,6 @@ + // SPDX-License-Identifier: GPL-2.0 + /* +- * Out-of-line refcount functions common to all refcount implementations. ++ * Out-of-line refcount functions. + */ + + #include +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-consolidate-refcount_-max-saturated.patch b/queue-5.4/locking-refcount-consolidate-refcount_-max-saturated.patch new file mode 100644 index 00000000000..19346589d7d --- /dev/null +++ b/queue-5.4/locking-refcount-consolidate-refcount_-max-saturated.patch @@ -0,0 +1,66 @@ +From d716a7dfb49329ead7158ad0e2e068e13e64c850 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:59 +0000 +Subject: locking/refcount: Consolidate REFCOUNT_{MAX,SATURATED} definitions + +From: Will Deacon + +[ Upstream commit 65b008552469f1c37f5e06e0016924502e40b4f5 ] + +The definitions of REFCOUNT_MAX and REFCOUNT_SATURATED are the same, +regardless of CONFIG_REFCOUNT_FULL, so consolidate them into a single +pair of definitions. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-8-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + include/linux/refcount.h | 9 ++------- + 1 file changed, 2 insertions(+), 7 deletions(-) + +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index 1cd0a876a789..757d4630115c 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -22,6 +22,8 @@ typedef struct refcount_struct { + } refcount_t; + + #define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), } ++#define REFCOUNT_MAX INT_MAX ++#define REFCOUNT_SATURATED (INT_MIN / 2) + + enum refcount_saturation_type { + REFCOUNT_ADD_NOT_ZERO_OVF, +@@ -57,9 +59,6 @@ static inline unsigned int refcount_read(const refcount_t *r) + #ifdef CONFIG_REFCOUNT_FULL + #include + +-#define REFCOUNT_MAX INT_MAX +-#define REFCOUNT_SATURATED (INT_MIN / 2) +- + /* + * Variant of atomic_t specialized for reference counts. + * +@@ -300,10 +299,6 @@ static inline void refcount_dec(refcount_t *r) + refcount_warn_saturate(r, REFCOUNT_DEC_LEAK); + } + #else /* CONFIG_REFCOUNT_FULL */ +- +-#define REFCOUNT_MAX INT_MAX +-#define REFCOUNT_SATURATED (INT_MIN / 2) +- + # ifdef CONFIG_ARCH_HAS_REFCOUNT + # include + # else +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-define-constants-for-saturation-and.patch b/queue-5.4/locking-refcount-define-constants-for-saturation-and.patch new file mode 100644 index 00000000000..1ea39067455 --- /dev/null +++ b/queue-5.4/locking-refcount-define-constants-for-saturation-and.patch @@ -0,0 +1,246 @@ +From bb7dc070751272c95bce37f086106d0d01b13be8 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:53 +0000 +Subject: locking/refcount: Define constants for saturation and max refcount + values + +From: Will Deacon + +[ Upstream commit 23e6b169c9917fbd77534f8c5f378cb073f548bd ] + +The REFCOUNT_FULL implementation uses a different saturation point than +the x86 implementation, which means that the shared refcount code in +lib/refcount.c (e.g. refcount_dec_not_one()) needs to be aware of the +difference. + +Rather than duplicate the definitions from the lkdtm driver, instead +move them into and update all references accordingly. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-2-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + drivers/misc/lkdtm/refcount.c | 8 -------- + include/linux/refcount.h | 10 +++++++++- + lib/refcount.c | 37 +++++++++++++++++++---------------- + 3 files changed, 29 insertions(+), 26 deletions(-) + +diff --git a/drivers/misc/lkdtm/refcount.c b/drivers/misc/lkdtm/refcount.c +index 0a146b32da13..abf3b7c1f686 100644 +--- a/drivers/misc/lkdtm/refcount.c ++++ b/drivers/misc/lkdtm/refcount.c +@@ -6,14 +6,6 @@ + #include "lkdtm.h" + #include + +-#ifdef CONFIG_REFCOUNT_FULL +-#define REFCOUNT_MAX (UINT_MAX - 1) +-#define REFCOUNT_SATURATED UINT_MAX +-#else +-#define REFCOUNT_MAX INT_MAX +-#define REFCOUNT_SATURATED (INT_MIN / 2) +-#endif +- + static void overflow_check(refcount_t *ref) + { + switch (refcount_read(ref)) { +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index e28cce21bad6..79f62e8d2256 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -4,6 +4,7 @@ + + #include + #include ++#include + #include + + struct mutex; +@@ -12,7 +13,7 @@ struct mutex; + * struct refcount_t - variant of atomic_t specialized for reference counts + * @refs: atomic_t counter field + * +- * The counter saturates at UINT_MAX and will not move once ++ * The counter saturates at REFCOUNT_SATURATED and will not move once + * there. This avoids wrapping the counter and causing 'spurious' + * use-after-free bugs. + */ +@@ -56,6 +57,9 @@ extern void refcount_dec_checked(refcount_t *r); + + #ifdef CONFIG_REFCOUNT_FULL + ++#define REFCOUNT_MAX (UINT_MAX - 1) ++#define REFCOUNT_SATURATED UINT_MAX ++ + #define refcount_add_not_zero refcount_add_not_zero_checked + #define refcount_add refcount_add_checked + +@@ -68,6 +72,10 @@ extern void refcount_dec_checked(refcount_t *r); + #define refcount_dec refcount_dec_checked + + #else ++ ++#define REFCOUNT_MAX INT_MAX ++#define REFCOUNT_SATURATED (INT_MIN / 2) ++ + # ifdef CONFIG_ARCH_HAS_REFCOUNT + # include + # else +diff --git a/lib/refcount.c b/lib/refcount.c +index 6e904af0fb3e..48b78a423d7d 100644 +--- a/lib/refcount.c ++++ b/lib/refcount.c +@@ -5,8 +5,8 @@ + * The interface matches the atomic_t interface (to aid in porting) but only + * provides the few functions one should use for reference counting. + * +- * It differs in that the counter saturates at UINT_MAX and will not move once +- * there. This avoids wrapping the counter and causing 'spurious' ++ * It differs in that the counter saturates at REFCOUNT_SATURATED and will not ++ * move once there. This avoids wrapping the counter and causing 'spurious' + * use-after-free issues. + * + * Memory ordering rules are slightly relaxed wrt regular atomic_t functions +@@ -48,7 +48,7 @@ + * @i: the value to add to the refcount + * @r: the refcount + * +- * Will saturate at UINT_MAX and WARN. ++ * Will saturate at REFCOUNT_SATURATED and WARN. + * + * Provides no memory ordering, it is assumed the caller has guaranteed the + * object memory to be stable (RCU, etc.). It does provide a control dependency +@@ -69,16 +69,17 @@ bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r) + if (!val) + return false; + +- if (unlikely(val == UINT_MAX)) ++ if (unlikely(val == REFCOUNT_SATURATED)) + return true; + + new = val + i; + if (new < val) +- new = UINT_MAX; ++ new = REFCOUNT_SATURATED; + + } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); + +- WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); ++ WARN_ONCE(new == REFCOUNT_SATURATED, ++ "refcount_t: saturated; leaking memory.\n"); + + return true; + } +@@ -89,7 +90,7 @@ EXPORT_SYMBOL(refcount_add_not_zero_checked); + * @i: the value to add to the refcount + * @r: the refcount + * +- * Similar to atomic_add(), but will saturate at UINT_MAX and WARN. ++ * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN. + * + * Provides no memory ordering, it is assumed the caller has guaranteed the + * object memory to be stable (RCU, etc.). It does provide a control dependency +@@ -110,7 +111,8 @@ EXPORT_SYMBOL(refcount_add_checked); + * refcount_inc_not_zero_checked - increment a refcount unless it is 0 + * @r: the refcount to increment + * +- * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN. ++ * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED ++ * and WARN. + * + * Provides no memory ordering, it is assumed the caller has guaranteed the + * object memory to be stable (RCU, etc.). It does provide a control dependency +@@ -133,7 +135,8 @@ bool refcount_inc_not_zero_checked(refcount_t *r) + + } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); + +- WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); ++ WARN_ONCE(new == REFCOUNT_SATURATED, ++ "refcount_t: saturated; leaking memory.\n"); + + return true; + } +@@ -143,7 +146,7 @@ EXPORT_SYMBOL(refcount_inc_not_zero_checked); + * refcount_inc_checked - increment a refcount + * @r: the refcount to increment + * +- * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN. ++ * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN. + * + * Provides no memory ordering, it is assumed the caller already has a + * reference on the object. +@@ -164,7 +167,7 @@ EXPORT_SYMBOL(refcount_inc_checked); + * + * Similar to atomic_dec_and_test(), but it will WARN, return false and + * ultimately leak on underflow and will fail to decrement when saturated +- * at UINT_MAX. ++ * at REFCOUNT_SATURATED. + * + * Provides release memory ordering, such that prior loads and stores are done + * before, and provides an acquire ordering on success such that free() +@@ -182,7 +185,7 @@ bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r) + unsigned int new, val = atomic_read(&r->refs); + + do { +- if (unlikely(val == UINT_MAX)) ++ if (unlikely(val == REFCOUNT_SATURATED)) + return false; + + new = val - i; +@@ -207,7 +210,7 @@ EXPORT_SYMBOL(refcount_sub_and_test_checked); + * @r: the refcount + * + * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to +- * decrement when saturated at UINT_MAX. ++ * decrement when saturated at REFCOUNT_SATURATED. + * + * Provides release memory ordering, such that prior loads and stores are done + * before, and provides an acquire ordering on success such that free() +@@ -226,7 +229,7 @@ EXPORT_SYMBOL(refcount_dec_and_test_checked); + * @r: the refcount + * + * Similar to atomic_dec(), it will WARN on underflow and fail to decrement +- * when saturated at UINT_MAX. ++ * when saturated at REFCOUNT_SATURATED. + * + * Provides release memory ordering, such that prior loads and stores are done + * before. +@@ -277,7 +280,7 @@ bool refcount_dec_not_one(refcount_t *r) + unsigned int new, val = atomic_read(&r->refs); + + do { +- if (unlikely(val == UINT_MAX)) ++ if (unlikely(val == REFCOUNT_SATURATED)) + return true; + + if (val == 1) +@@ -302,7 +305,7 @@ EXPORT_SYMBOL(refcount_dec_not_one); + * @lock: the mutex to be locked + * + * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail +- * to decrement when saturated at UINT_MAX. ++ * to decrement when saturated at REFCOUNT_SATURATED. + * + * Provides release memory ordering, such that prior loads and stores are done + * before, and provides a control dependency such that free() must come after. +@@ -333,7 +336,7 @@ EXPORT_SYMBOL(refcount_dec_and_mutex_lock); + * @lock: the spinlock to be locked + * + * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to +- * decrement when saturated at UINT_MAX. ++ * decrement when saturated at REFCOUNT_SATURATED. + * + * Provides release memory ordering, such that prior loads and stores are done + * before, and provides a control dependency such that free() must come after. +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-ensure-integer-operands-are-treated.patch b/queue-5.4/locking-refcount-ensure-integer-operands-are-treated.patch new file mode 100644 index 00000000000..5b229515926 --- /dev/null +++ b/queue-5.4/locking-refcount-ensure-integer-operands-are-treated.patch @@ -0,0 +1,125 @@ +From db84554aa58fd3d15d6299589bd50987c70612b7 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:54 +0000 +Subject: locking/refcount: Ensure integer operands are treated as signed + +From: Will Deacon + +[ Upstream commit 97a1420adf0cdf0cf6f41bab0b2acf658c96b94b ] + +In preparation for changing the saturation point of REFCOUNT_FULL to +INT_MIN/2, change the type of integer operands passed into the API +from 'unsigned int' to 'int' so that we can avoid casting during +comparisons when we don't want to fall foul of C integral conversion +rules for signed and unsigned types. + +Since the kernel is compiled with '-fno-strict-overflow', we don't need +to worry about the UB introduced by signed overflow here. Furthermore, +we're already making heavy use of the atomic_t API, which operates +exclusively on signed types. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-3-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + include/linux/refcount.h | 14 +++++++------- + lib/refcount.c | 6 +++--- + 2 files changed, 10 insertions(+), 10 deletions(-) + +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index 79f62e8d2256..89066a1471dd 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -28,7 +28,7 @@ typedef struct refcount_struct { + * @r: the refcount + * @n: value to which the refcount will be set + */ +-static inline void refcount_set(refcount_t *r, unsigned int n) ++static inline void refcount_set(refcount_t *r, int n) + { + atomic_set(&r->refs, n); + } +@@ -44,13 +44,13 @@ static inline unsigned int refcount_read(const refcount_t *r) + return atomic_read(&r->refs); + } + +-extern __must_check bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r); +-extern void refcount_add_checked(unsigned int i, refcount_t *r); ++extern __must_check bool refcount_add_not_zero_checked(int i, refcount_t *r); ++extern void refcount_add_checked(int i, refcount_t *r); + + extern __must_check bool refcount_inc_not_zero_checked(refcount_t *r); + extern void refcount_inc_checked(refcount_t *r); + +-extern __must_check bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r); ++extern __must_check bool refcount_sub_and_test_checked(int i, refcount_t *r); + + extern __must_check bool refcount_dec_and_test_checked(refcount_t *r); + extern void refcount_dec_checked(refcount_t *r); +@@ -79,12 +79,12 @@ extern void refcount_dec_checked(refcount_t *r); + # ifdef CONFIG_ARCH_HAS_REFCOUNT + # include + # else +-static inline __must_check bool refcount_add_not_zero(unsigned int i, refcount_t *r) ++static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) + { + return atomic_add_unless(&r->refs, i, 0); + } + +-static inline void refcount_add(unsigned int i, refcount_t *r) ++static inline void refcount_add(int i, refcount_t *r) + { + atomic_add(i, &r->refs); + } +@@ -99,7 +99,7 @@ static inline void refcount_inc(refcount_t *r) + atomic_inc(&r->refs); + } + +-static inline __must_check bool refcount_sub_and_test(unsigned int i, refcount_t *r) ++static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) + { + return atomic_sub_and_test(i, &r->refs); + } +diff --git a/lib/refcount.c b/lib/refcount.c +index 48b78a423d7d..719b0bc42ab1 100644 +--- a/lib/refcount.c ++++ b/lib/refcount.c +@@ -61,7 +61,7 @@ + * + * Return: false if the passed refcount is 0, true otherwise + */ +-bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r) ++bool refcount_add_not_zero_checked(int i, refcount_t *r) + { + unsigned int new, val = atomic_read(&r->refs); + +@@ -101,7 +101,7 @@ EXPORT_SYMBOL(refcount_add_not_zero_checked); + * cases, refcount_inc(), or one of its variants, should instead be used to + * increment a reference count. + */ +-void refcount_add_checked(unsigned int i, refcount_t *r) ++void refcount_add_checked(int i, refcount_t *r) + { + WARN_ONCE(!refcount_add_not_zero_checked(i, r), "refcount_t: addition on 0; use-after-free.\n"); + } +@@ -180,7 +180,7 @@ EXPORT_SYMBOL(refcount_inc_checked); + * + * Return: true if the resulting refcount is 0, false otherwise + */ +-bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r) ++bool refcount_sub_and_test_checked(int i, refcount_t *r) + { + unsigned int new, val = atomic_read(&r->refs); + +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-improve-performance-of-generic-refc.patch b/queue-5.4/locking-refcount-improve-performance-of-generic-refc.patch new file mode 100644 index 00000000000..d50933fc3e2 --- /dev/null +++ b/queue-5.4/locking-refcount-improve-performance-of-generic-refc.patch @@ -0,0 +1,251 @@ +From 6eaed59065030285985ed945780aacb7dec937cd Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:57 +0000 +Subject: locking/refcount: Improve performance of generic REFCOUNT_FULL code + +From: Will Deacon + +[ Upstream commit dcb786493f3e48da3272b710028d42ec608cfda1 ] + +Rewrite the generic REFCOUNT_FULL implementation so that the saturation +point is moved to INT_MIN / 2. This allows us to defer the sanity checks +until after the atomic operation, which removes many uses of cmpxchg() +in favour of atomic_fetch_{add,sub}(). + +Some crude perf results obtained from lkdtm show substantially less +overhead, despite the checking: + + $ perf stat -r 3 -B -- echo {ATOMIC,REFCOUNT}_TIMING >/sys/kernel/debug/provoke-crash/DIRECT + + # arm64 + ATOMIC_TIMING: 46.50451 +- 0.00134 seconds time elapsed ( +- 0.00% ) + REFCOUNT_TIMING (REFCOUNT_FULL, mainline): 77.57522 +- 0.00982 seconds time elapsed ( +- 0.01% ) + REFCOUNT_TIMING (REFCOUNT_FULL, this series): 48.7181 +- 0.0256 seconds time elapsed ( +- 0.05% ) + + # x86 + ATOMIC_TIMING: 31.6225 +- 0.0776 seconds time elapsed ( +- 0.25% ) + REFCOUNT_TIMING (!REFCOUNT_FULL, mainline/x86 asm): 31.6689 +- 0.0901 seconds time elapsed ( +- 0.28% ) + REFCOUNT_TIMING (REFCOUNT_FULL, mainline): 53.203 +- 0.138 seconds time elapsed ( +- 0.26% ) + REFCOUNT_TIMING (REFCOUNT_FULL, this series): 31.7408 +- 0.0486 seconds time elapsed ( +- 0.15% ) + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Tested-by: Jan Glauber +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-6-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + include/linux/refcount.h | 131 ++++++++++++++++++++++----------------- + 1 file changed, 75 insertions(+), 56 deletions(-) + +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index e719b5b1220e..e3b218d669ce 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -47,8 +47,8 @@ static inline unsigned int refcount_read(const refcount_t *r) + #ifdef CONFIG_REFCOUNT_FULL + #include + +-#define REFCOUNT_MAX (UINT_MAX - 1) +-#define REFCOUNT_SATURATED UINT_MAX ++#define REFCOUNT_MAX INT_MAX ++#define REFCOUNT_SATURATED (INT_MIN / 2) + + /* + * Variant of atomic_t specialized for reference counts. +@@ -56,9 +56,47 @@ static inline unsigned int refcount_read(const refcount_t *r) + * The interface matches the atomic_t interface (to aid in porting) but only + * provides the few functions one should use for reference counting. + * +- * It differs in that the counter saturates at REFCOUNT_SATURATED and will not +- * move once there. This avoids wrapping the counter and causing 'spurious' +- * use-after-free issues. ++ * Saturation semantics ++ * ==================== ++ * ++ * refcount_t differs from atomic_t in that the counter saturates at ++ * REFCOUNT_SATURATED and will not move once there. This avoids wrapping the ++ * counter and causing 'spurious' use-after-free issues. In order to avoid the ++ * cost associated with introducing cmpxchg() loops into all of the saturating ++ * operations, we temporarily allow the counter to take on an unchecked value ++ * and then explicitly set it to REFCOUNT_SATURATED on detecting that underflow ++ * or overflow has occurred. Although this is racy when multiple threads ++ * access the refcount concurrently, by placing REFCOUNT_SATURATED roughly ++ * equidistant from 0 and INT_MAX we minimise the scope for error: ++ * ++ * INT_MAX REFCOUNT_SATURATED UINT_MAX ++ * 0 (0x7fff_ffff) (0xc000_0000) (0xffff_ffff) ++ * +--------------------------------+----------------+----------------+ ++ * <---------- bad value! ----------> ++ * ++ * (in a signed view of the world, the "bad value" range corresponds to ++ * a negative counter value). ++ * ++ * As an example, consider a refcount_inc() operation that causes the counter ++ * to overflow: ++ * ++ * int old = atomic_fetch_add_relaxed(r); ++ * // old is INT_MAX, refcount now INT_MIN (0x8000_0000) ++ * if (old < 0) ++ * atomic_set(r, REFCOUNT_SATURATED); ++ * ++ * If another thread also performs a refcount_inc() operation between the two ++ * atomic operations, then the count will continue to edge closer to 0. If it ++ * reaches a value of 1 before /any/ of the threads reset it to the saturated ++ * value, then a concurrent refcount_dec_and_test() may erroneously free the ++ * underlying object. Given the precise timing details involved with the ++ * round-robin scheduling of each thread manipulating the refcount and the need ++ * to hit the race multiple times in succession, there doesn't appear to be a ++ * practical avenue of attack even if using refcount_add() operations with ++ * larger increments. ++ * ++ * Memory ordering ++ * =============== + * + * Memory ordering rules are slightly relaxed wrt regular atomic_t functions + * and provide only what is strictly required for refcounts. +@@ -109,25 +147,19 @@ static inline unsigned int refcount_read(const refcount_t *r) + */ + static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) + { +- unsigned int new, val = atomic_read(&r->refs); ++ int old = refcount_read(r); + + do { +- if (!val) +- return false; +- +- if (unlikely(val == REFCOUNT_SATURATED)) +- return true; +- +- new = val + i; +- if (new < val) +- new = REFCOUNT_SATURATED; ++ if (!old) ++ break; ++ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i)); + +- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); +- +- WARN_ONCE(new == REFCOUNT_SATURATED, +- "refcount_t: saturated; leaking memory.\n"); ++ if (unlikely(old < 0 || old + i < 0)) { ++ refcount_set(r, REFCOUNT_SATURATED); ++ WARN_ONCE(1, "refcount_t: saturated; leaking memory.\n"); ++ } + +- return true; ++ return old; + } + + /** +@@ -148,7 +180,13 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) + */ + static inline void refcount_add(int i, refcount_t *r) + { +- WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n"); ++ int old = atomic_fetch_add_relaxed(i, &r->refs); ++ ++ WARN_ONCE(!old, "refcount_t: addition on 0; use-after-free.\n"); ++ if (unlikely(old <= 0 || old + i <= 0)) { ++ refcount_set(r, REFCOUNT_SATURATED); ++ WARN_ONCE(old, "refcount_t: saturated; leaking memory.\n"); ++ } + } + + /** +@@ -166,23 +204,7 @@ static inline void refcount_add(int i, refcount_t *r) + */ + static inline __must_check bool refcount_inc_not_zero(refcount_t *r) + { +- unsigned int new, val = atomic_read(&r->refs); +- +- do { +- new = val + 1; +- +- if (!val) +- return false; +- +- if (unlikely(!new)) +- return true; +- +- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); +- +- WARN_ONCE(new == REFCOUNT_SATURATED, +- "refcount_t: saturated; leaking memory.\n"); +- +- return true; ++ return refcount_add_not_zero(1, r); + } + + /** +@@ -199,7 +221,7 @@ static inline __must_check bool refcount_inc_not_zero(refcount_t *r) + */ + static inline void refcount_inc(refcount_t *r) + { +- WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n"); ++ refcount_add(1, r); + } + + /** +@@ -224,26 +246,19 @@ static inline void refcount_inc(refcount_t *r) + */ + static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) + { +- unsigned int new, val = atomic_read(&r->refs); +- +- do { +- if (unlikely(val == REFCOUNT_SATURATED)) +- return false; ++ int old = atomic_fetch_sub_release(i, &r->refs); + +- new = val - i; +- if (new > val) { +- WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); +- return false; +- } +- +- } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); +- +- if (!new) { ++ if (old == i) { + smp_acquire__after_ctrl_dep(); + return true; + } +- return false; + ++ if (unlikely(old < 0 || old - i < 0)) { ++ refcount_set(r, REFCOUNT_SATURATED); ++ WARN_ONCE(1, "refcount_t: underflow; use-after-free.\n"); ++ } ++ ++ return false; + } + + /** +@@ -276,9 +291,13 @@ static inline __must_check bool refcount_dec_and_test(refcount_t *r) + */ + static inline void refcount_dec(refcount_t *r) + { +- WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n"); +-} ++ int old = atomic_fetch_sub_release(1, &r->refs); + ++ if (unlikely(old <= 1)) { ++ refcount_set(r, REFCOUNT_SATURATED); ++ WARN_ONCE(1, "refcount_t: decrement hit 0; leaking memory.\n"); ++ } ++} + #else /* CONFIG_REFCOUNT_FULL */ + + #define REFCOUNT_MAX INT_MAX +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-move-saturation-warnings-out-of-lin.patch b/queue-5.4/locking-refcount-move-saturation-warnings-out-of-lin.patch new file mode 100644 index 00000000000..26167b31818 --- /dev/null +++ b/queue-5.4/locking-refcount-move-saturation-warnings-out-of-lin.patch @@ -0,0 +1,163 @@ +From 4692c5514cef668722dfaeab60870fe90ec9f978 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:58 +0000 +Subject: locking/refcount: Move saturation warnings out of line + +From: Will Deacon + +[ Upstream commit 1eb085d94256aaa69b00cf5a86e3c5f5bb2bc460 ] + +Having the refcount saturation and warnings inline bloats the text, +despite the fact that these paths should never be executed in normal +operation. + +Move the refcount saturation and warnings out of line to reduce the +image size when refcount_t checking is enabled. Relative to an x86_64 +defconfig, the sizes reported by bloat-o-meter are: + + # defconfig+REFCOUNT_FULL, inline saturation (i.e. before this patch) + Total: Before=14762076, After=14915442, chg +1.04% + + # defconfig+REFCOUNT_FULL, out-of-line saturation (i.e. after this patch) + Total: Before=14762076, After=14835497, chg +0.50% + +A side-effect of this change is that we now only get one warning per +refcount saturation type, rather than one per problematic call-site. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-7-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + include/linux/refcount.h | 39 ++++++++++++++++++++------------------- + lib/refcount.c | 28 ++++++++++++++++++++++++++++ + 2 files changed, 48 insertions(+), 19 deletions(-) + +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index e3b218d669ce..1cd0a876a789 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -23,6 +23,16 @@ typedef struct refcount_struct { + + #define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), } + ++enum refcount_saturation_type { ++ REFCOUNT_ADD_NOT_ZERO_OVF, ++ REFCOUNT_ADD_OVF, ++ REFCOUNT_ADD_UAF, ++ REFCOUNT_SUB_UAF, ++ REFCOUNT_DEC_LEAK, ++}; ++ ++void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t); ++ + /** + * refcount_set - set a refcount's value + * @r: the refcount +@@ -154,10 +164,8 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) + break; + } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i)); + +- if (unlikely(old < 0 || old + i < 0)) { +- refcount_set(r, REFCOUNT_SATURATED); +- WARN_ONCE(1, "refcount_t: saturated; leaking memory.\n"); +- } ++ if (unlikely(old < 0 || old + i < 0)) ++ refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF); + + return old; + } +@@ -182,11 +190,10 @@ static inline void refcount_add(int i, refcount_t *r) + { + int old = atomic_fetch_add_relaxed(i, &r->refs); + +- WARN_ONCE(!old, "refcount_t: addition on 0; use-after-free.\n"); +- if (unlikely(old <= 0 || old + i <= 0)) { +- refcount_set(r, REFCOUNT_SATURATED); +- WARN_ONCE(old, "refcount_t: saturated; leaking memory.\n"); +- } ++ if (unlikely(!old)) ++ refcount_warn_saturate(r, REFCOUNT_ADD_UAF); ++ else if (unlikely(old < 0 || old + i < 0)) ++ refcount_warn_saturate(r, REFCOUNT_ADD_OVF); + } + + /** +@@ -253,10 +260,8 @@ static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) + return true; + } + +- if (unlikely(old < 0 || old - i < 0)) { +- refcount_set(r, REFCOUNT_SATURATED); +- WARN_ONCE(1, "refcount_t: underflow; use-after-free.\n"); +- } ++ if (unlikely(old < 0 || old - i < 0)) ++ refcount_warn_saturate(r, REFCOUNT_SUB_UAF); + + return false; + } +@@ -291,12 +296,8 @@ static inline __must_check bool refcount_dec_and_test(refcount_t *r) + */ + static inline void refcount_dec(refcount_t *r) + { +- int old = atomic_fetch_sub_release(1, &r->refs); +- +- if (unlikely(old <= 1)) { +- refcount_set(r, REFCOUNT_SATURATED); +- WARN_ONCE(1, "refcount_t: decrement hit 0; leaking memory.\n"); +- } ++ if (unlikely(atomic_fetch_sub_release(1, &r->refs) <= 1)) ++ refcount_warn_saturate(r, REFCOUNT_DEC_LEAK); + } + #else /* CONFIG_REFCOUNT_FULL */ + +diff --git a/lib/refcount.c b/lib/refcount.c +index 3a534fbebdcc..8b7e249c0e10 100644 +--- a/lib/refcount.c ++++ b/lib/refcount.c +@@ -8,6 +8,34 @@ + #include + #include + ++#define REFCOUNT_WARN(str) WARN_ONCE(1, "refcount_t: " str ".\n") ++ ++void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t) ++{ ++ refcount_set(r, REFCOUNT_SATURATED); ++ ++ switch (t) { ++ case REFCOUNT_ADD_NOT_ZERO_OVF: ++ REFCOUNT_WARN("saturated; leaking memory"); ++ break; ++ case REFCOUNT_ADD_OVF: ++ REFCOUNT_WARN("saturated; leaking memory"); ++ break; ++ case REFCOUNT_ADD_UAF: ++ REFCOUNT_WARN("addition on 0; use-after-free"); ++ break; ++ case REFCOUNT_SUB_UAF: ++ REFCOUNT_WARN("underflow; use-after-free"); ++ break; ++ case REFCOUNT_DEC_LEAK: ++ REFCOUNT_WARN("decrement hit 0; leaking memory"); ++ break; ++ default: ++ REFCOUNT_WARN("unknown saturation event!?"); ++ } ++} ++EXPORT_SYMBOL(refcount_warn_saturate); ++ + /** + * refcount_dec_if_one - decrement a refcount if it is 1 + * @r: the refcount +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-move-the-bulk-of-the-refcount_full-.patch b/queue-5.4/locking-refcount-move-the-bulk-of-the-refcount_full-.patch new file mode 100644 index 00000000000..e8c5e558056 --- /dev/null +++ b/queue-5.4/locking-refcount-move-the-bulk-of-the-refcount_full-.patch @@ -0,0 +1,553 @@ +From 27c24b3cffb5963450ebc988eee4780cce1961ea Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:56 +0000 +Subject: locking/refcount: Move the bulk of the REFCOUNT_FULL implementation + into the header + +From: Will Deacon + +[ Upstream commit 77e9971c79c29542ab7dd4140f9343bf2ff36158 ] + +In an effort to improve performance of the REFCOUNT_FULL implementation, +move the bulk of its functions into linux/refcount.h. This allows them +to be inlined in the same way as if they had been provided via +CONFIG_ARCH_HAS_REFCOUNT. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-5-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + include/linux/refcount.h | 237 ++++++++++++++++++++++++++++++++++++-- + lib/refcount.c | 238 +-------------------------------------- + 2 files changed, 229 insertions(+), 246 deletions(-) + +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index edd505d1a23b..e719b5b1220e 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -45,22 +45,241 @@ static inline unsigned int refcount_read(const refcount_t *r) + } + + #ifdef CONFIG_REFCOUNT_FULL ++#include + + #define REFCOUNT_MAX (UINT_MAX - 1) + #define REFCOUNT_SATURATED UINT_MAX + +-extern __must_check bool refcount_add_not_zero(int i, refcount_t *r); +-extern void refcount_add(int i, refcount_t *r); ++/* ++ * Variant of atomic_t specialized for reference counts. ++ * ++ * The interface matches the atomic_t interface (to aid in porting) but only ++ * provides the few functions one should use for reference counting. ++ * ++ * It differs in that the counter saturates at REFCOUNT_SATURATED and will not ++ * move once there. This avoids wrapping the counter and causing 'spurious' ++ * use-after-free issues. ++ * ++ * Memory ordering rules are slightly relaxed wrt regular atomic_t functions ++ * and provide only what is strictly required for refcounts. ++ * ++ * The increments are fully relaxed; these will not provide ordering. The ++ * rationale is that whatever is used to obtain the object we're increasing the ++ * reference count on will provide the ordering. For locked data structures, ++ * its the lock acquire, for RCU/lockless data structures its the dependent ++ * load. ++ * ++ * Do note that inc_not_zero() provides a control dependency which will order ++ * future stores against the inc, this ensures we'll never modify the object ++ * if we did not in fact acquire a reference. ++ * ++ * The decrements will provide release order, such that all the prior loads and ++ * stores will be issued before, it also provides a control dependency, which ++ * will order us against the subsequent free(). ++ * ++ * The control dependency is against the load of the cmpxchg (ll/sc) that ++ * succeeded. This means the stores aren't fully ordered, but this is fine ++ * because the 1->0 transition indicates no concurrency. ++ * ++ * Note that the allocator is responsible for ordering things between free() ++ * and alloc(). ++ * ++ * The decrements dec_and_test() and sub_and_test() also provide acquire ++ * ordering on success. ++ * ++ */ ++ ++/** ++ * refcount_add_not_zero - add a value to a refcount unless it is 0 ++ * @i: the value to add to the refcount ++ * @r: the refcount ++ * ++ * Will saturate at REFCOUNT_SATURATED and WARN. ++ * ++ * Provides no memory ordering, it is assumed the caller has guaranteed the ++ * object memory to be stable (RCU, etc.). It does provide a control dependency ++ * and thereby orders future stores. See the comment on top. ++ * ++ * Use of this function is not recommended for the normal reference counting ++ * use case in which references are taken and released one at a time. In these ++ * cases, refcount_inc(), or one of its variants, should instead be used to ++ * increment a reference count. ++ * ++ * Return: false if the passed refcount is 0, true otherwise ++ */ ++static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) ++{ ++ unsigned int new, val = atomic_read(&r->refs); ++ ++ do { ++ if (!val) ++ return false; ++ ++ if (unlikely(val == REFCOUNT_SATURATED)) ++ return true; ++ ++ new = val + i; ++ if (new < val) ++ new = REFCOUNT_SATURATED; ++ ++ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); ++ ++ WARN_ONCE(new == REFCOUNT_SATURATED, ++ "refcount_t: saturated; leaking memory.\n"); ++ ++ return true; ++} ++ ++/** ++ * refcount_add - add a value to a refcount ++ * @i: the value to add to the refcount ++ * @r: the refcount ++ * ++ * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN. ++ * ++ * Provides no memory ordering, it is assumed the caller has guaranteed the ++ * object memory to be stable (RCU, etc.). It does provide a control dependency ++ * and thereby orders future stores. See the comment on top. ++ * ++ * Use of this function is not recommended for the normal reference counting ++ * use case in which references are taken and released one at a time. In these ++ * cases, refcount_inc(), or one of its variants, should instead be used to ++ * increment a reference count. ++ */ ++static inline void refcount_add(int i, refcount_t *r) ++{ ++ WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n"); ++} ++ ++/** ++ * refcount_inc_not_zero - increment a refcount unless it is 0 ++ * @r: the refcount to increment ++ * ++ * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED ++ * and WARN. ++ * ++ * Provides no memory ordering, it is assumed the caller has guaranteed the ++ * object memory to be stable (RCU, etc.). It does provide a control dependency ++ * and thereby orders future stores. See the comment on top. ++ * ++ * Return: true if the increment was successful, false otherwise ++ */ ++static inline __must_check bool refcount_inc_not_zero(refcount_t *r) ++{ ++ unsigned int new, val = atomic_read(&r->refs); ++ ++ do { ++ new = val + 1; + +-extern __must_check bool refcount_inc_not_zero(refcount_t *r); +-extern void refcount_inc(refcount_t *r); ++ if (!val) ++ return false; + +-extern __must_check bool refcount_sub_and_test(int i, refcount_t *r); ++ if (unlikely(!new)) ++ return true; + +-extern __must_check bool refcount_dec_and_test(refcount_t *r); +-extern void refcount_dec(refcount_t *r); ++ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); ++ ++ WARN_ONCE(new == REFCOUNT_SATURATED, ++ "refcount_t: saturated; leaking memory.\n"); ++ ++ return true; ++} ++ ++/** ++ * refcount_inc - increment a refcount ++ * @r: the refcount to increment ++ * ++ * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN. ++ * ++ * Provides no memory ordering, it is assumed the caller already has a ++ * reference on the object. ++ * ++ * Will WARN if the refcount is 0, as this represents a possible use-after-free ++ * condition. ++ */ ++static inline void refcount_inc(refcount_t *r) ++{ ++ WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n"); ++} ++ ++/** ++ * refcount_sub_and_test - subtract from a refcount and test if it is 0 ++ * @i: amount to subtract from the refcount ++ * @r: the refcount ++ * ++ * Similar to atomic_dec_and_test(), but it will WARN, return false and ++ * ultimately leak on underflow and will fail to decrement when saturated ++ * at REFCOUNT_SATURATED. ++ * ++ * Provides release memory ordering, such that prior loads and stores are done ++ * before, and provides an acquire ordering on success such that free() ++ * must come after. ++ * ++ * Use of this function is not recommended for the normal reference counting ++ * use case in which references are taken and released one at a time. In these ++ * cases, refcount_dec(), or one of its variants, should instead be used to ++ * decrement a reference count. ++ * ++ * Return: true if the resulting refcount is 0, false otherwise ++ */ ++static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) ++{ ++ unsigned int new, val = atomic_read(&r->refs); ++ ++ do { ++ if (unlikely(val == REFCOUNT_SATURATED)) ++ return false; ++ ++ new = val - i; ++ if (new > val) { ++ WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); ++ return false; ++ } ++ ++ } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); ++ ++ if (!new) { ++ smp_acquire__after_ctrl_dep(); ++ return true; ++ } ++ return false; ++ ++} ++ ++/** ++ * refcount_dec_and_test - decrement a refcount and test if it is 0 ++ * @r: the refcount ++ * ++ * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to ++ * decrement when saturated at REFCOUNT_SATURATED. ++ * ++ * Provides release memory ordering, such that prior loads and stores are done ++ * before, and provides an acquire ordering on success such that free() ++ * must come after. ++ * ++ * Return: true if the resulting refcount is 0, false otherwise ++ */ ++static inline __must_check bool refcount_dec_and_test(refcount_t *r) ++{ ++ return refcount_sub_and_test(1, r); ++} ++ ++/** ++ * refcount_dec - decrement a refcount ++ * @r: the refcount ++ * ++ * Similar to atomic_dec(), it will WARN on underflow and fail to decrement ++ * when saturated at REFCOUNT_SATURATED. ++ * ++ * Provides release memory ordering, such that prior loads and stores are done ++ * before. ++ */ ++static inline void refcount_dec(refcount_t *r) ++{ ++ WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n"); ++} + +-#else ++#else /* CONFIG_REFCOUNT_FULL */ + + #define REFCOUNT_MAX INT_MAX + #define REFCOUNT_SATURATED (INT_MIN / 2) +@@ -103,7 +322,7 @@ static inline void refcount_dec(refcount_t *r) + atomic_dec(&r->refs); + } + # endif /* !CONFIG_ARCH_HAS_REFCOUNT */ +-#endif /* CONFIG_REFCOUNT_FULL */ ++#endif /* !CONFIG_REFCOUNT_FULL */ + + extern __must_check bool refcount_dec_if_one(refcount_t *r); + extern __must_check bool refcount_dec_not_one(refcount_t *r); +diff --git a/lib/refcount.c b/lib/refcount.c +index a2f670998cee..3a534fbebdcc 100644 +--- a/lib/refcount.c ++++ b/lib/refcount.c +@@ -1,41 +1,6 @@ + // SPDX-License-Identifier: GPL-2.0 + /* +- * Variant of atomic_t specialized for reference counts. +- * +- * The interface matches the atomic_t interface (to aid in porting) but only +- * provides the few functions one should use for reference counting. +- * +- * It differs in that the counter saturates at REFCOUNT_SATURATED and will not +- * move once there. This avoids wrapping the counter and causing 'spurious' +- * use-after-free issues. +- * +- * Memory ordering rules are slightly relaxed wrt regular atomic_t functions +- * and provide only what is strictly required for refcounts. +- * +- * The increments are fully relaxed; these will not provide ordering. The +- * rationale is that whatever is used to obtain the object we're increasing the +- * reference count on will provide the ordering. For locked data structures, +- * its the lock acquire, for RCU/lockless data structures its the dependent +- * load. +- * +- * Do note that inc_not_zero() provides a control dependency which will order +- * future stores against the inc, this ensures we'll never modify the object +- * if we did not in fact acquire a reference. +- * +- * The decrements will provide release order, such that all the prior loads and +- * stores will be issued before, it also provides a control dependency, which +- * will order us against the subsequent free(). +- * +- * The control dependency is against the load of the cmpxchg (ll/sc) that +- * succeeded. This means the stores aren't fully ordered, but this is fine +- * because the 1->0 transition indicates no concurrency. +- * +- * Note that the allocator is responsible for ordering things between free() +- * and alloc(). +- * +- * The decrements dec_and_test() and sub_and_test() also provide acquire +- * ordering on success. +- * ++ * Out-of-line refcount functions common to all refcount implementations. + */ + + #include +@@ -43,207 +8,6 @@ + #include + #include + +-#ifdef CONFIG_REFCOUNT_FULL +- +-/** +- * refcount_add_not_zero - add a value to a refcount unless it is 0 +- * @i: the value to add to the refcount +- * @r: the refcount +- * +- * Will saturate at REFCOUNT_SATURATED and WARN. +- * +- * Provides no memory ordering, it is assumed the caller has guaranteed the +- * object memory to be stable (RCU, etc.). It does provide a control dependency +- * and thereby orders future stores. See the comment on top. +- * +- * Use of this function is not recommended for the normal reference counting +- * use case in which references are taken and released one at a time. In these +- * cases, refcount_inc(), or one of its variants, should instead be used to +- * increment a reference count. +- * +- * Return: false if the passed refcount is 0, true otherwise +- */ +-bool refcount_add_not_zero(int i, refcount_t *r) +-{ +- unsigned int new, val = atomic_read(&r->refs); +- +- do { +- if (!val) +- return false; +- +- if (unlikely(val == REFCOUNT_SATURATED)) +- return true; +- +- new = val + i; +- if (new < val) +- new = REFCOUNT_SATURATED; +- +- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); +- +- WARN_ONCE(new == REFCOUNT_SATURATED, +- "refcount_t: saturated; leaking memory.\n"); +- +- return true; +-} +-EXPORT_SYMBOL(refcount_add_not_zero); +- +-/** +- * refcount_add - add a value to a refcount +- * @i: the value to add to the refcount +- * @r: the refcount +- * +- * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN. +- * +- * Provides no memory ordering, it is assumed the caller has guaranteed the +- * object memory to be stable (RCU, etc.). It does provide a control dependency +- * and thereby orders future stores. See the comment on top. +- * +- * Use of this function is not recommended for the normal reference counting +- * use case in which references are taken and released one at a time. In these +- * cases, refcount_inc(), or one of its variants, should instead be used to +- * increment a reference count. +- */ +-void refcount_add(int i, refcount_t *r) +-{ +- WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n"); +-} +-EXPORT_SYMBOL(refcount_add); +- +-/** +- * refcount_inc_not_zero - increment a refcount unless it is 0 +- * @r: the refcount to increment +- * +- * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED +- * and WARN. +- * +- * Provides no memory ordering, it is assumed the caller has guaranteed the +- * object memory to be stable (RCU, etc.). It does provide a control dependency +- * and thereby orders future stores. See the comment on top. +- * +- * Return: true if the increment was successful, false otherwise +- */ +-bool refcount_inc_not_zero(refcount_t *r) +-{ +- unsigned int new, val = atomic_read(&r->refs); +- +- do { +- new = val + 1; +- +- if (!val) +- return false; +- +- if (unlikely(!new)) +- return true; +- +- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); +- +- WARN_ONCE(new == REFCOUNT_SATURATED, +- "refcount_t: saturated; leaking memory.\n"); +- +- return true; +-} +-EXPORT_SYMBOL(refcount_inc_not_zero); +- +-/** +- * refcount_inc - increment a refcount +- * @r: the refcount to increment +- * +- * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN. +- * +- * Provides no memory ordering, it is assumed the caller already has a +- * reference on the object. +- * +- * Will WARN if the refcount is 0, as this represents a possible use-after-free +- * condition. +- */ +-void refcount_inc(refcount_t *r) +-{ +- WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n"); +-} +-EXPORT_SYMBOL(refcount_inc); +- +-/** +- * refcount_sub_and_test - subtract from a refcount and test if it is 0 +- * @i: amount to subtract from the refcount +- * @r: the refcount +- * +- * Similar to atomic_dec_and_test(), but it will WARN, return false and +- * ultimately leak on underflow and will fail to decrement when saturated +- * at REFCOUNT_SATURATED. +- * +- * Provides release memory ordering, such that prior loads and stores are done +- * before, and provides an acquire ordering on success such that free() +- * must come after. +- * +- * Use of this function is not recommended for the normal reference counting +- * use case in which references are taken and released one at a time. In these +- * cases, refcount_dec(), or one of its variants, should instead be used to +- * decrement a reference count. +- * +- * Return: true if the resulting refcount is 0, false otherwise +- */ +-bool refcount_sub_and_test(int i, refcount_t *r) +-{ +- unsigned int new, val = atomic_read(&r->refs); +- +- do { +- if (unlikely(val == REFCOUNT_SATURATED)) +- return false; +- +- new = val - i; +- if (new > val) { +- WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); +- return false; +- } +- +- } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); +- +- if (!new) { +- smp_acquire__after_ctrl_dep(); +- return true; +- } +- return false; +- +-} +-EXPORT_SYMBOL(refcount_sub_and_test); +- +-/** +- * refcount_dec_and_test - decrement a refcount and test if it is 0 +- * @r: the refcount +- * +- * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to +- * decrement when saturated at REFCOUNT_SATURATED. +- * +- * Provides release memory ordering, such that prior loads and stores are done +- * before, and provides an acquire ordering on success such that free() +- * must come after. +- * +- * Return: true if the resulting refcount is 0, false otherwise +- */ +-bool refcount_dec_and_test(refcount_t *r) +-{ +- return refcount_sub_and_test(1, r); +-} +-EXPORT_SYMBOL(refcount_dec_and_test); +- +-/** +- * refcount_dec - decrement a refcount +- * @r: the refcount +- * +- * Similar to atomic_dec(), it will WARN on underflow and fail to decrement +- * when saturated at REFCOUNT_SATURATED. +- * +- * Provides release memory ordering, such that prior loads and stores are done +- * before. +- */ +-void refcount_dec(refcount_t *r) +-{ +- WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n"); +-} +-EXPORT_SYMBOL(refcount_dec); +- +-#endif /* CONFIG_REFCOUNT_FULL */ +- + /** + * refcount_dec_if_one - decrement a refcount if it is 1 + * @r: the refcount +-- +2.35.1 + diff --git a/queue-5.4/locking-refcount-remove-unused-refcount_-_checked-va.patch b/queue-5.4/locking-refcount-remove-unused-refcount_-_checked-va.patch new file mode 100644 index 00000000000..ca3ae521aaa --- /dev/null +++ b/queue-5.4/locking-refcount-remove-unused-refcount_-_checked-va.patch @@ -0,0 +1,238 @@ +From b820cde8559e0882bd6688bc9a5497bd7b2b13ec Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Thu, 21 Nov 2019 11:58:55 +0000 +Subject: locking/refcount: Remove unused refcount_*_checked() variants + +From: Will Deacon + +[ Upstream commit 7221762c48c6bbbcc6cc51d8b803c06930215e34 ] + +The full-fat refcount implementation is exposed via a set of functions +suffixed with "_checked()", the idea being that code can choose to use +the more expensive, yet more secure implementation on a case-by-case +basis. + +In reality, this hasn't happened, so with a grand total of zero users, +let's remove the checked variants for now by simply dropping the suffix +and predicating the out-of-line functions on CONFIG_REFCOUNT_FULL=y. + +Signed-off-by: Will Deacon +Reviewed-by: Ard Biesheuvel +Reviewed-by: Kees Cook +Tested-by: Hanjun Guo +Cc: Ard Biesheuvel +Cc: Elena Reshetova +Cc: Linus Torvalds +Cc: Peter Zijlstra +Cc: Thomas Gleixner +Link: https://lkml.kernel.org/r/20191121115902.2551-4-will@kernel.org +Signed-off-by: Ingo Molnar +Signed-off-by: Sasha Levin +--- + include/linux/refcount.h | 25 ++++++------------- + lib/refcount.c | 54 +++++++++++++++++++++------------------- + 2 files changed, 36 insertions(+), 43 deletions(-) + +diff --git a/include/linux/refcount.h b/include/linux/refcount.h +index 89066a1471dd..edd505d1a23b 100644 +--- a/include/linux/refcount.h ++++ b/include/linux/refcount.h +@@ -44,32 +44,21 @@ static inline unsigned int refcount_read(const refcount_t *r) + return atomic_read(&r->refs); + } + +-extern __must_check bool refcount_add_not_zero_checked(int i, refcount_t *r); +-extern void refcount_add_checked(int i, refcount_t *r); +- +-extern __must_check bool refcount_inc_not_zero_checked(refcount_t *r); +-extern void refcount_inc_checked(refcount_t *r); +- +-extern __must_check bool refcount_sub_and_test_checked(int i, refcount_t *r); +- +-extern __must_check bool refcount_dec_and_test_checked(refcount_t *r); +-extern void refcount_dec_checked(refcount_t *r); +- + #ifdef CONFIG_REFCOUNT_FULL + + #define REFCOUNT_MAX (UINT_MAX - 1) + #define REFCOUNT_SATURATED UINT_MAX + +-#define refcount_add_not_zero refcount_add_not_zero_checked +-#define refcount_add refcount_add_checked ++extern __must_check bool refcount_add_not_zero(int i, refcount_t *r); ++extern void refcount_add(int i, refcount_t *r); + +-#define refcount_inc_not_zero refcount_inc_not_zero_checked +-#define refcount_inc refcount_inc_checked ++extern __must_check bool refcount_inc_not_zero(refcount_t *r); ++extern void refcount_inc(refcount_t *r); + +-#define refcount_sub_and_test refcount_sub_and_test_checked ++extern __must_check bool refcount_sub_and_test(int i, refcount_t *r); + +-#define refcount_dec_and_test refcount_dec_and_test_checked +-#define refcount_dec refcount_dec_checked ++extern __must_check bool refcount_dec_and_test(refcount_t *r); ++extern void refcount_dec(refcount_t *r); + + #else + +diff --git a/lib/refcount.c b/lib/refcount.c +index 719b0bc42ab1..a2f670998cee 100644 +--- a/lib/refcount.c ++++ b/lib/refcount.c +@@ -43,8 +43,10 @@ + #include + #include + ++#ifdef CONFIG_REFCOUNT_FULL ++ + /** +- * refcount_add_not_zero_checked - add a value to a refcount unless it is 0 ++ * refcount_add_not_zero - add a value to a refcount unless it is 0 + * @i: the value to add to the refcount + * @r: the refcount + * +@@ -61,7 +63,7 @@ + * + * Return: false if the passed refcount is 0, true otherwise + */ +-bool refcount_add_not_zero_checked(int i, refcount_t *r) ++bool refcount_add_not_zero(int i, refcount_t *r) + { + unsigned int new, val = atomic_read(&r->refs); + +@@ -83,10 +85,10 @@ bool refcount_add_not_zero_checked(int i, refcount_t *r) + + return true; + } +-EXPORT_SYMBOL(refcount_add_not_zero_checked); ++EXPORT_SYMBOL(refcount_add_not_zero); + + /** +- * refcount_add_checked - add a value to a refcount ++ * refcount_add - add a value to a refcount + * @i: the value to add to the refcount + * @r: the refcount + * +@@ -101,14 +103,14 @@ EXPORT_SYMBOL(refcount_add_not_zero_checked); + * cases, refcount_inc(), or one of its variants, should instead be used to + * increment a reference count. + */ +-void refcount_add_checked(int i, refcount_t *r) ++void refcount_add(int i, refcount_t *r) + { +- WARN_ONCE(!refcount_add_not_zero_checked(i, r), "refcount_t: addition on 0; use-after-free.\n"); ++ WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n"); + } +-EXPORT_SYMBOL(refcount_add_checked); ++EXPORT_SYMBOL(refcount_add); + + /** +- * refcount_inc_not_zero_checked - increment a refcount unless it is 0 ++ * refcount_inc_not_zero - increment a refcount unless it is 0 + * @r: the refcount to increment + * + * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED +@@ -120,7 +122,7 @@ EXPORT_SYMBOL(refcount_add_checked); + * + * Return: true if the increment was successful, false otherwise + */ +-bool refcount_inc_not_zero_checked(refcount_t *r) ++bool refcount_inc_not_zero(refcount_t *r) + { + unsigned int new, val = atomic_read(&r->refs); + +@@ -140,10 +142,10 @@ bool refcount_inc_not_zero_checked(refcount_t *r) + + return true; + } +-EXPORT_SYMBOL(refcount_inc_not_zero_checked); ++EXPORT_SYMBOL(refcount_inc_not_zero); + + /** +- * refcount_inc_checked - increment a refcount ++ * refcount_inc - increment a refcount + * @r: the refcount to increment + * + * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN. +@@ -154,14 +156,14 @@ EXPORT_SYMBOL(refcount_inc_not_zero_checked); + * Will WARN if the refcount is 0, as this represents a possible use-after-free + * condition. + */ +-void refcount_inc_checked(refcount_t *r) ++void refcount_inc(refcount_t *r) + { +- WARN_ONCE(!refcount_inc_not_zero_checked(r), "refcount_t: increment on 0; use-after-free.\n"); ++ WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n"); + } +-EXPORT_SYMBOL(refcount_inc_checked); ++EXPORT_SYMBOL(refcount_inc); + + /** +- * refcount_sub_and_test_checked - subtract from a refcount and test if it is 0 ++ * refcount_sub_and_test - subtract from a refcount and test if it is 0 + * @i: amount to subtract from the refcount + * @r: the refcount + * +@@ -180,7 +182,7 @@ EXPORT_SYMBOL(refcount_inc_checked); + * + * Return: true if the resulting refcount is 0, false otherwise + */ +-bool refcount_sub_and_test_checked(int i, refcount_t *r) ++bool refcount_sub_and_test(int i, refcount_t *r) + { + unsigned int new, val = atomic_read(&r->refs); + +@@ -203,10 +205,10 @@ bool refcount_sub_and_test_checked(int i, refcount_t *r) + return false; + + } +-EXPORT_SYMBOL(refcount_sub_and_test_checked); ++EXPORT_SYMBOL(refcount_sub_and_test); + + /** +- * refcount_dec_and_test_checked - decrement a refcount and test if it is 0 ++ * refcount_dec_and_test - decrement a refcount and test if it is 0 + * @r: the refcount + * + * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to +@@ -218,14 +220,14 @@ EXPORT_SYMBOL(refcount_sub_and_test_checked); + * + * Return: true if the resulting refcount is 0, false otherwise + */ +-bool refcount_dec_and_test_checked(refcount_t *r) ++bool refcount_dec_and_test(refcount_t *r) + { +- return refcount_sub_and_test_checked(1, r); ++ return refcount_sub_and_test(1, r); + } +-EXPORT_SYMBOL(refcount_dec_and_test_checked); ++EXPORT_SYMBOL(refcount_dec_and_test); + + /** +- * refcount_dec_checked - decrement a refcount ++ * refcount_dec - decrement a refcount + * @r: the refcount + * + * Similar to atomic_dec(), it will WARN on underflow and fail to decrement +@@ -234,11 +236,13 @@ EXPORT_SYMBOL(refcount_dec_and_test_checked); + * Provides release memory ordering, such that prior loads and stores are done + * before. + */ +-void refcount_dec_checked(refcount_t *r) ++void refcount_dec(refcount_t *r) + { +- WARN_ONCE(refcount_dec_and_test_checked(r), "refcount_t: decrement hit 0; leaking memory.\n"); ++ WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n"); + } +-EXPORT_SYMBOL(refcount_dec_checked); ++EXPORT_SYMBOL(refcount_dec); ++ ++#endif /* CONFIG_REFCOUNT_FULL */ + + /** + * refcount_dec_if_one - decrement a refcount if it is 1 +-- +2.35.1 + diff --git a/queue-5.4/mmap-locking-api-initial-implementation-as-rwsem-wra.patch b/queue-5.4/mmap-locking-api-initial-implementation-as-rwsem-wra.patch new file mode 100644 index 00000000000..a5fbab0c966 --- /dev/null +++ b/queue-5.4/mmap-locking-api-initial-implementation-as-rwsem-wra.patch @@ -0,0 +1,136 @@ +From fb0bf383598b6cdb897a7cb05abeb8a61963f27e Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Mon, 8 Jun 2020 21:33:14 -0700 +Subject: mmap locking API: initial implementation as rwsem wrappers + +From: Michel Lespinasse + +[ Upstream commit 9740ca4e95b43b91a4a848694a20d01ba6818f7b ] + +This patch series adds a new mmap locking API replacing the existing +mmap_sem lock and unlocks. Initially the API is just implemente in terms +of inlined rwsem calls, so it doesn't provide any new functionality. + +There are two justifications for the new API: + +- At first, it provides an easy hooking point to instrument mmap_sem + locking latencies independently of any other rwsems. + +- In the future, it may be a starting point for replacing the rwsem + implementation with a different one, such as range locks. This is + something that is being explored, even though there is no wide concensus + about this possible direction yet. (see + https://patchwork.kernel.org/cover/11401483/) + +This patch (of 12): + +This change wraps the existing mmap_sem related rwsem calls into a new +mmap locking API. There are two justifications for the new API: + +- At first, it provides an easy hooking point to instrument mmap_sem + locking latencies independently of any other rwsems. + +- In the future, it may be a starting point for replacing the rwsem + implementation with a different one, such as range locks. + +Signed-off-by: Michel Lespinasse +Signed-off-by: Andrew Morton +Reviewed-by: Daniel Jordan +Reviewed-by: Davidlohr Bueso +Reviewed-by: Laurent Dufour +Reviewed-by: Vlastimil Babka +Cc: Peter Zijlstra +Cc: Matthew Wilcox +Cc: Liam Howlett +Cc: Jerome Glisse +Cc: David Rientjes +Cc: Hugh Dickins +Cc: Ying Han +Cc: Jason Gunthorpe +Cc: John Hubbard +Cc: Michel Lespinasse +Link: http://lkml.kernel.org/r/20200520052908.204642-1-walken@google.com +Link: http://lkml.kernel.org/r/20200520052908.204642-2-walken@google.com +Signed-off-by: Linus Torvalds +Signed-off-by: Sasha Levin +--- + include/linux/mm.h | 1 + + include/linux/mmap_lock.h | 54 +++++++++++++++++++++++++++++++++++++++ + 2 files changed, 55 insertions(+) + create mode 100644 include/linux/mmap_lock.h + +diff --git a/include/linux/mm.h b/include/linux/mm.h +index c125fea49752..d35c29d322d8 100644 +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h +new file mode 100644 +index 000000000000..97ac53b66052 +--- /dev/null ++++ b/include/linux/mmap_lock.h +@@ -0,0 +1,54 @@ ++#ifndef _LINUX_MMAP_LOCK_H ++#define _LINUX_MMAP_LOCK_H ++ ++static inline void mmap_init_lock(struct mm_struct *mm) ++{ ++ init_rwsem(&mm->mmap_sem); ++} ++ ++static inline void mmap_write_lock(struct mm_struct *mm) ++{ ++ down_write(&mm->mmap_sem); ++} ++ ++static inline int mmap_write_lock_killable(struct mm_struct *mm) ++{ ++ return down_write_killable(&mm->mmap_sem); ++} ++ ++static inline bool mmap_write_trylock(struct mm_struct *mm) ++{ ++ return down_write_trylock(&mm->mmap_sem) != 0; ++} ++ ++static inline void mmap_write_unlock(struct mm_struct *mm) ++{ ++ up_write(&mm->mmap_sem); ++} ++ ++static inline void mmap_write_downgrade(struct mm_struct *mm) ++{ ++ downgrade_write(&mm->mmap_sem); ++} ++ ++static inline void mmap_read_lock(struct mm_struct *mm) ++{ ++ down_read(&mm->mmap_sem); ++} ++ ++static inline int mmap_read_lock_killable(struct mm_struct *mm) ++{ ++ return down_read_killable(&mm->mmap_sem); ++} ++ ++static inline bool mmap_read_trylock(struct mm_struct *mm) ++{ ++ return down_read_trylock(&mm->mmap_sem) != 0; ++} ++ ++static inline void mmap_read_unlock(struct mm_struct *mm) ++{ ++ up_read(&mm->mmap_sem); ++} ++ ++#endif /* _LINUX_MMAP_LOCK_H */ +-- +2.35.1 + diff --git a/queue-5.4/series b/queue-5.4/series index 2f66a707878..bbc6feeb0e1 100644 --- a/queue-5.4/series +++ b/queue-5.4/series @@ -55,3 +55,19 @@ tcp-fix-data-races-around-sysctl_tcp_max_reordering.patch spi-bcm2835-bcm2835_spi_handle_err-fix-null-pointer-deref-for-non-dma-transfers.patch mm-mempolicy-fix-uninit-value-in-mpol_rebind_policy.patch bpf-make-sure-mac_header-was-set-before-using-it.patch +dlm-fix-pending-remove-if-msg-allocation-fails.patch +ima-remove-the-ima_template-kconfig-option.patch +locking-refcount-define-constants-for-saturation-and.patch +locking-refcount-ensure-integer-operands-are-treated.patch +locking-refcount-remove-unused-refcount_-_checked-va.patch +locking-refcount-move-the-bulk-of-the-refcount_full-.patch +locking-refcount-improve-performance-of-generic-refc.patch +locking-refcount-move-saturation-warnings-out-of-lin.patch +locking-refcount-consolidate-refcount_-max-saturated.patch +locking-refcount-consolidate-implementations-of-refc.patch +x86-get-rid-of-small-constant-size-cases-in-raw_copy.patch +x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch +x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch-104 +mmap-locking-api-initial-implementation-as-rwsem-wra.patch +x86-mce-deduplicate-exception-handling.patch +bitfield.h-fix-type-of-reg-too-small-for-mask-test.patch diff --git a/queue-5.4/x86-get-rid-of-small-constant-size-cases-in-raw_copy.patch b/queue-5.4/x86-get-rid-of-small-constant-size-cases-in-raw_copy.patch new file mode 100644 index 00000000000..5ebdc809607 --- /dev/null +++ b/queue-5.4/x86-get-rid-of-small-constant-size-cases-in-raw_copy.patch @@ -0,0 +1,209 @@ +From 9978eea9fcfec3bdfb3157ef9ce6b3badb31c5f6 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Sat, 15 Feb 2020 11:46:30 -0500 +Subject: x86: get rid of small constant size cases in + raw_copy_{to,from}_user() + +From: Al Viro + +[ Upstream commit 4b842e4e25b12951fa10dedb4bc16bc47e3b850c ] + +Very few call sites where that would be triggered remain, and none +of those is anywhere near hot enough to bother. + +Signed-off-by: Al Viro +Signed-off-by: Sasha Levin +--- + arch/x86/include/asm/uaccess.h | 12 ---- + arch/x86/include/asm/uaccess_32.h | 27 -------- + arch/x86/include/asm/uaccess_64.h | 108 +----------------------------- + 3 files changed, 2 insertions(+), 145 deletions(-) + +diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h +index 61d93f062a36..a19effb98fdc 100644 +--- a/arch/x86/include/asm/uaccess.h ++++ b/arch/x86/include/asm/uaccess.h +@@ -378,18 +378,6 @@ do { \ + : "=r" (err), ltype(x) \ + : "m" (__m(addr)), "i" (errret), "0" (err)) + +-#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \ +- asm volatile("\n" \ +- "1: mov"itype" %2,%"rtype"1\n" \ +- "2:\n" \ +- ".section .fixup,\"ax\"\n" \ +- "3: mov %3,%0\n" \ +- " jmp 2b\n" \ +- ".previous\n" \ +- _ASM_EXTABLE_UA(1b, 3b) \ +- : "=r" (err), ltype(x) \ +- : "m" (__m(addr)), "i" (errret), "0" (err)) +- + /* + * This doesn't do __uaccess_begin/end - the exception handling + * around it must do that. +diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h +index ba2dc1930630..388a40660c7b 100644 +--- a/arch/x86/include/asm/uaccess_32.h ++++ b/arch/x86/include/asm/uaccess_32.h +@@ -23,33 +23,6 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) + static __always_inline unsigned long + raw_copy_from_user(void *to, const void __user *from, unsigned long n) + { +- if (__builtin_constant_p(n)) { +- unsigned long ret; +- +- switch (n) { +- case 1: +- ret = 0; +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u8 *)to, from, ret, +- "b", "b", "=q", 1); +- __uaccess_end(); +- return ret; +- case 2: +- ret = 0; +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u16 *)to, from, ret, +- "w", "w", "=r", 2); +- __uaccess_end(); +- return ret; +- case 4: +- ret = 0; +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u32 *)to, from, ret, +- "l", "k", "=r", 4); +- __uaccess_end(); +- return ret; +- } +- } + return __copy_user_ll(to, (__force const void *)from, n); + } + +diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h +index 5cd1caa8bc65..bc10e3dc64fe 100644 +--- a/arch/x86/include/asm/uaccess_64.h ++++ b/arch/x86/include/asm/uaccess_64.h +@@ -65,117 +65,13 @@ copy_to_user_mcsafe(void *to, const void *from, unsigned len) + static __always_inline __must_check unsigned long + raw_copy_from_user(void *dst, const void __user *src, unsigned long size) + { +- int ret = 0; +- +- if (!__builtin_constant_p(size)) +- return copy_user_generic(dst, (__force void *)src, size); +- switch (size) { +- case 1: +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, +- ret, "b", "b", "=q", 1); +- __uaccess_end(); +- return ret; +- case 2: +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, +- ret, "w", "w", "=r", 2); +- __uaccess_end(); +- return ret; +- case 4: +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, +- ret, "l", "k", "=r", 4); +- __uaccess_end(); +- return ret; +- case 8: +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, +- ret, "q", "", "=r", 8); +- __uaccess_end(); +- return ret; +- case 10: +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, +- ret, "q", "", "=r", 10); +- if (likely(!ret)) +- __get_user_asm_nozero(*(u16 *)(8 + (char *)dst), +- (u16 __user *)(8 + (char __user *)src), +- ret, "w", "w", "=r", 2); +- __uaccess_end(); +- return ret; +- case 16: +- __uaccess_begin_nospec(); +- __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, +- ret, "q", "", "=r", 16); +- if (likely(!ret)) +- __get_user_asm_nozero(*(u64 *)(8 + (char *)dst), +- (u64 __user *)(8 + (char __user *)src), +- ret, "q", "", "=r", 8); +- __uaccess_end(); +- return ret; +- default: +- return copy_user_generic(dst, (__force void *)src, size); +- } ++ return copy_user_generic(dst, (__force void *)src, size); + } + + static __always_inline __must_check unsigned long + raw_copy_to_user(void __user *dst, const void *src, unsigned long size) + { +- int ret = 0; +- +- if (!__builtin_constant_p(size)) +- return copy_user_generic((__force void *)dst, src, size); +- switch (size) { +- case 1: +- __uaccess_begin(); +- __put_user_asm(*(u8 *)src, (u8 __user *)dst, +- ret, "b", "b", "iq", 1); +- __uaccess_end(); +- return ret; +- case 2: +- __uaccess_begin(); +- __put_user_asm(*(u16 *)src, (u16 __user *)dst, +- ret, "w", "w", "ir", 2); +- __uaccess_end(); +- return ret; +- case 4: +- __uaccess_begin(); +- __put_user_asm(*(u32 *)src, (u32 __user *)dst, +- ret, "l", "k", "ir", 4); +- __uaccess_end(); +- return ret; +- case 8: +- __uaccess_begin(); +- __put_user_asm(*(u64 *)src, (u64 __user *)dst, +- ret, "q", "", "er", 8); +- __uaccess_end(); +- return ret; +- case 10: +- __uaccess_begin(); +- __put_user_asm(*(u64 *)src, (u64 __user *)dst, +- ret, "q", "", "er", 10); +- if (likely(!ret)) { +- asm("":::"memory"); +- __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, +- ret, "w", "w", "ir", 2); +- } +- __uaccess_end(); +- return ret; +- case 16: +- __uaccess_begin(); +- __put_user_asm(*(u64 *)src, (u64 __user *)dst, +- ret, "q", "", "er", 16); +- if (likely(!ret)) { +- asm("":::"memory"); +- __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, +- ret, "q", "", "er", 8); +- } +- __uaccess_end(); +- return ret; +- default: +- return copy_user_generic((__force void *)dst, src, size); +- } ++ return copy_user_generic((__force void *)dst, src, size); + } + + static __always_inline __must_check +-- +2.35.1 + diff --git a/queue-5.4/x86-mce-deduplicate-exception-handling.patch b/queue-5.4/x86-mce-deduplicate-exception-handling.patch new file mode 100644 index 00000000000..a6fae497411 --- /dev/null +++ b/queue-5.4/x86-mce-deduplicate-exception-handling.patch @@ -0,0 +1,83 @@ +From d83ef6366175fc15938a0f610f7131ba9e7a2eb3 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Wed, 8 Sep 2021 15:29:15 +0200 +Subject: x86/mce: Deduplicate exception handling + +From: Thomas Gleixner + +[ Upstream commit e42404afc4ca856c48f1e05752541faa3587c472 ] + +Prepare code for further simplification. No functional change. + +Signed-off-by: Thomas Gleixner +Signed-off-by: Borislav Petkov +Link: https://lkml.kernel.org/r/20210908132525.096452100@linutronix.de +Signed-off-by: Sasha Levin +--- + arch/x86/kernel/cpu/mce/core.c | 34 +++++++++++++++++----------------- + 1 file changed, 17 insertions(+), 17 deletions(-) + +diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c +index 8a2b8e791314..9b98a7d8ac60 100644 +--- a/arch/x86/kernel/cpu/mce/core.c ++++ b/arch/x86/kernel/cpu/mce/core.c +@@ -397,13 +397,16 @@ static int msr_to_offset(u32 msr) + return -1; + } + +-__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, +- struct pt_regs *regs, int trapnr, +- unsigned long error_code, +- unsigned long fault_addr) ++static void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) + { +- pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", +- (unsigned int)regs->cx, regs->ip, (void *)regs->ip); ++ if (wrmsr) { ++ pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", ++ (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, ++ regs->ip, (void *)regs->ip); ++ } else { ++ pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", ++ (unsigned int)regs->cx, regs->ip, (void *)regs->ip); ++ } + + show_stack_regs(regs); + +@@ -411,7 +414,14 @@ __visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, + + while (true) + cpu_relax(); ++} + ++__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, ++ struct pt_regs *regs, int trapnr, ++ unsigned long error_code, ++ unsigned long fault_addr) ++{ ++ ex_handler_msr_mce(regs, false); + return true; + } + +@@ -447,17 +457,7 @@ __visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup, + unsigned long error_code, + unsigned long fault_addr) + { +- pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", +- (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, +- regs->ip, (void *)regs->ip); +- +- show_stack_regs(regs); +- +- panic("MCA architectural violation!\n"); +- +- while (true) +- cpu_relax(); +- ++ ex_handler_msr_mce(regs, true); + return true; + } + +-- +2.35.1 + diff --git a/queue-5.4/x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch b/queue-5.4/x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch new file mode 100644 index 00000000000..afc801bdfde --- /dev/null +++ b/queue-5.4/x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch @@ -0,0 +1,196 @@ +From d5f813d6f5eef4f0b5e40233ef3484fb8d973a31 Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Wed, 2 Feb 2022 00:49:42 +0000 +Subject: x86/uaccess: Implement macros for CMPXCHG on user addresses + +From: Peter Zijlstra + +[ Upstream commit 989b5db215a2f22f89d730b607b071d964780f10 ] + +Add support for CMPXCHG loops on userspace addresses. Provide both an +"unsafe" version for tight loops that do their own uaccess begin/end, as +well as a "safe" version for use cases where the CMPXCHG is not buried in +a loop, e.g. KVM will resume the guest instead of looping when emulation +of a guest atomic accesses fails the CMPXCHG. + +Provide 8-byte versions for 32-bit kernels so that KVM can do CMPXCHG on +guest PAE PTEs, which are accessed via userspace addresses. + +Guard the asm_volatile_goto() variation with CC_HAS_ASM_GOTO_TIED_OUTPUT, +the "+m" constraint fails on some compilers that otherwise support +CC_HAS_ASM_GOTO_OUTPUT. + +Cc: stable@vger.kernel.org +Signed-off-by: Peter Zijlstra (Intel) +Co-developed-by: Sean Christopherson +Signed-off-by: Sean Christopherson +Message-Id: <20220202004945.2540433-3-seanjc@google.com> +Signed-off-by: Paolo Bonzini +Signed-off-by: Sasha Levin +--- + arch/x86/include/asm/uaccess.h | 142 +++++++++++++++++++++++++++++++++ + 1 file changed, 142 insertions(+) + +diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h +index a19effb98fdc..865795e2355e 100644 +--- a/arch/x86/include/asm/uaccess.h ++++ b/arch/x86/include/asm/uaccess.h +@@ -441,6 +441,103 @@ __pu_label: \ + __builtin_expect(__gu_err, 0); \ + }) + ++#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT ++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \ ++ bool success; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm_volatile_goto("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ ++ _ASM_EXTABLE_UA(1b, %l[label]) \ ++ : CC_OUT(z) (success), \ ++ [ptr] "+m" (*_ptr), \ ++ [old] "+a" (__old) \ ++ : [new] ltype (__new) \ ++ : "memory" \ ++ : label); \ ++ if (unlikely(!success)) \ ++ *_old = __old; \ ++ likely(success); }) ++ ++#ifdef CONFIG_X86_32 ++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \ ++ bool success; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm_volatile_goto("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ ++ _ASM_EXTABLE_UA(1b, %l[label]) \ ++ : CC_OUT(z) (success), \ ++ "+A" (__old), \ ++ [ptr] "+m" (*_ptr) \ ++ : "b" ((u32)__new), \ ++ "c" ((u32)((u64)__new >> 32)) \ ++ : "memory" \ ++ : label); \ ++ if (unlikely(!success)) \ ++ *_old = __old; \ ++ likely(success); }) ++#endif // CONFIG_X86_32 ++#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT ++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \ ++ int __err = 0; \ ++ bool success; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm volatile("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ ++ CC_SET(z) \ ++ "2:\n" \ ++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \ ++ %[errout]) \ ++ : CC_OUT(z) (success), \ ++ [errout] "+r" (__err), \ ++ [ptr] "+m" (*_ptr), \ ++ [old] "+a" (__old) \ ++ : [new] ltype (__new) \ ++ : "memory", "cc"); \ ++ if (unlikely(__err)) \ ++ goto label; \ ++ if (unlikely(!success)) \ ++ *_old = __old; \ ++ likely(success); }) ++ ++#ifdef CONFIG_X86_32 ++/* ++ * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error. ++ * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are ++ * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses ++ * both ESI and EDI for the memory operand, compilation will fail if the error ++ * is an input+output as there will be no register available for input. ++ */ ++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \ ++ int __result; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm volatile("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ ++ "mov $0, %%ecx\n\t" \ ++ "setz %%cl\n" \ ++ "2:\n" \ ++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \ ++ : [result]"=c" (__result), \ ++ "+A" (__old), \ ++ [ptr] "+m" (*_ptr) \ ++ : "b" ((u32)__new), \ ++ "c" ((u32)((u64)__new >> 32)) \ ++ : "memory", "cc"); \ ++ if (unlikely(__result < 0)) \ ++ goto label; \ ++ if (unlikely(!__result)) \ ++ *_old = __old; \ ++ likely(__result); }) ++#endif // CONFIG_X86_32 ++#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT ++ + /* FIXME: this hack is definitely wrong -AK */ + struct __large_struct { unsigned long buf[100]; }; + #define __m(x) (*(struct __large_struct __user *)(x)) +@@ -722,6 +819,51 @@ do { \ + if (unlikely(__gu_err)) goto err_label; \ + } while (0) + ++extern void __try_cmpxchg_user_wrong_size(void); ++ ++#ifndef CONFIG_X86_32 ++#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \ ++ __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label) ++#endif ++ ++/* ++ * Force the pointer to u to match the size expected by the asm helper. ++ * clang/LLVM compiles all cases and only discards the unused paths after ++ * processing errors, which breaks i386 if the pointer is an 8-byte value. ++ */ ++#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \ ++ bool __ret; \ ++ __chk_user_ptr(_ptr); \ ++ switch (sizeof(*(_ptr))) { \ ++ case 1: __ret = __try_cmpxchg_user_asm("b", "q", \ ++ (__force u8 *)(_ptr), (_oldp), \ ++ (_nval), _label); \ ++ break; \ ++ case 2: __ret = __try_cmpxchg_user_asm("w", "r", \ ++ (__force u16 *)(_ptr), (_oldp), \ ++ (_nval), _label); \ ++ break; \ ++ case 4: __ret = __try_cmpxchg_user_asm("l", "r", \ ++ (__force u32 *)(_ptr), (_oldp), \ ++ (_nval), _label); \ ++ break; \ ++ case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\ ++ (_nval), _label); \ ++ break; \ ++ default: __try_cmpxchg_user_wrong_size(); \ ++ } \ ++ __ret; }) ++ ++/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */ ++#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \ ++ int __ret = -EFAULT; \ ++ __uaccess_begin_nospec(); \ ++ __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \ ++_label: \ ++ __uaccess_end(); \ ++ __ret; \ ++ }) ++ + /* + * We want the unsafe accessors to always be inlined and use + * the error labels - thus the macro games. +-- +2.35.1 + diff --git a/queue-5.4/x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch-104 b/queue-5.4/x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch-104 new file mode 100644 index 00000000000..0e0c0f63a01 --- /dev/null +++ b/queue-5.4/x86-uaccess-implement-macros-for-cmpxchg-on-user-add.patch-104 @@ -0,0 +1,196 @@ +From b64be47e3d6b941b1934109ad7388ef0748547fc Mon Sep 17 00:00:00 2001 +From: Sasha Levin +Date: Wed, 2 Feb 2022 00:49:42 +0000 +Subject: x86/uaccess: Implement macros for CMPXCHG on user addresses + +From: Peter Zijlstra + +[ Upstream commit 989b5db215a2f22f89d730b607b071d964780f10 ] + +Add support for CMPXCHG loops on userspace addresses. Provide both an +"unsafe" version for tight loops that do their own uaccess begin/end, as +well as a "safe" version for use cases where the CMPXCHG is not buried in +a loop, e.g. KVM will resume the guest instead of looping when emulation +of a guest atomic accesses fails the CMPXCHG. + +Provide 8-byte versions for 32-bit kernels so that KVM can do CMPXCHG on +guest PAE PTEs, which are accessed via userspace addresses. + +Guard the asm_volatile_goto() variation with CC_HAS_ASM_GOTO_TIED_OUTPUT, +the "+m" constraint fails on some compilers that otherwise support +CC_HAS_ASM_GOTO_OUTPUT. + +Cc: stable@vger.kernel.org +Signed-off-by: Peter Zijlstra (Intel) +Co-developed-by: Sean Christopherson +Signed-off-by: Sean Christopherson +Message-Id: <20220202004945.2540433-3-seanjc@google.com> +Signed-off-by: Paolo Bonzini +Signed-off-by: Sasha Levin +--- + arch/x86/include/asm/uaccess.h | 142 +++++++++++++++++++++++++++++++++ + 1 file changed, 142 insertions(+) + +diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h +index 865795e2355e..7299acbbb7a6 100644 +--- a/arch/x86/include/asm/uaccess.h ++++ b/arch/x86/include/asm/uaccess.h +@@ -538,6 +538,103 @@ __pu_label: \ + #endif // CONFIG_X86_32 + #endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT + ++#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT ++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \ ++ bool success; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm_volatile_goto("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ ++ _ASM_EXTABLE_UA(1b, %l[label]) \ ++ : CC_OUT(z) (success), \ ++ [ptr] "+m" (*_ptr), \ ++ [old] "+a" (__old) \ ++ : [new] ltype (__new) \ ++ : "memory" \ ++ : label); \ ++ if (unlikely(!success)) \ ++ *_old = __old; \ ++ likely(success); }) ++ ++#ifdef CONFIG_X86_32 ++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \ ++ bool success; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm_volatile_goto("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ ++ _ASM_EXTABLE_UA(1b, %l[label]) \ ++ : CC_OUT(z) (success), \ ++ "+A" (__old), \ ++ [ptr] "+m" (*_ptr) \ ++ : "b" ((u32)__new), \ ++ "c" ((u32)((u64)__new >> 32)) \ ++ : "memory" \ ++ : label); \ ++ if (unlikely(!success)) \ ++ *_old = __old; \ ++ likely(success); }) ++#endif // CONFIG_X86_32 ++#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT ++#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \ ++ int __err = 0; \ ++ bool success; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm volatile("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ ++ CC_SET(z) \ ++ "2:\n" \ ++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \ ++ %[errout]) \ ++ : CC_OUT(z) (success), \ ++ [errout] "+r" (__err), \ ++ [ptr] "+m" (*_ptr), \ ++ [old] "+a" (__old) \ ++ : [new] ltype (__new) \ ++ : "memory", "cc"); \ ++ if (unlikely(__err)) \ ++ goto label; \ ++ if (unlikely(!success)) \ ++ *_old = __old; \ ++ likely(success); }) ++ ++#ifdef CONFIG_X86_32 ++/* ++ * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error. ++ * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are ++ * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses ++ * both ESI and EDI for the memory operand, compilation will fail if the error ++ * is an input+output as there will be no register available for input. ++ */ ++#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \ ++ int __result; \ ++ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ ++ __typeof__(*(_ptr)) __old = *_old; \ ++ __typeof__(*(_ptr)) __new = (_new); \ ++ asm volatile("\n" \ ++ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ ++ "mov $0, %%ecx\n\t" \ ++ "setz %%cl\n" \ ++ "2:\n" \ ++ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \ ++ : [result]"=c" (__result), \ ++ "+A" (__old), \ ++ [ptr] "+m" (*_ptr) \ ++ : "b" ((u32)__new), \ ++ "c" ((u32)((u64)__new >> 32)) \ ++ : "memory", "cc"); \ ++ if (unlikely(__result < 0)) \ ++ goto label; \ ++ if (unlikely(!__result)) \ ++ *_old = __old; \ ++ likely(__result); }) ++#endif // CONFIG_X86_32 ++#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT ++ + /* FIXME: this hack is definitely wrong -AK */ + struct __large_struct { unsigned long buf[100]; }; + #define __m(x) (*(struct __large_struct __user *)(x)) +@@ -826,6 +923,51 @@ extern void __try_cmpxchg_user_wrong_size(void); + __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label) + #endif + ++/* ++ * Force the pointer to u to match the size expected by the asm helper. ++ * clang/LLVM compiles all cases and only discards the unused paths after ++ * processing errors, which breaks i386 if the pointer is an 8-byte value. ++ */ ++#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \ ++ bool __ret; \ ++ __chk_user_ptr(_ptr); \ ++ switch (sizeof(*(_ptr))) { \ ++ case 1: __ret = __try_cmpxchg_user_asm("b", "q", \ ++ (__force u8 *)(_ptr), (_oldp), \ ++ (_nval), _label); \ ++ break; \ ++ case 2: __ret = __try_cmpxchg_user_asm("w", "r", \ ++ (__force u16 *)(_ptr), (_oldp), \ ++ (_nval), _label); \ ++ break; \ ++ case 4: __ret = __try_cmpxchg_user_asm("l", "r", \ ++ (__force u32 *)(_ptr), (_oldp), \ ++ (_nval), _label); \ ++ break; \ ++ case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\ ++ (_nval), _label); \ ++ break; \ ++ default: __try_cmpxchg_user_wrong_size(); \ ++ } \ ++ __ret; }) ++ ++/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */ ++#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \ ++ int __ret = -EFAULT; \ ++ __uaccess_begin_nospec(); \ ++ __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \ ++_label: \ ++ __uaccess_end(); \ ++ __ret; \ ++ }) ++ ++extern void __try_cmpxchg_user_wrong_size(void); ++ ++#ifndef CONFIG_X86_32 ++#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \ ++ __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label) ++#endif ++ + /* + * Force the pointer to u to match the size expected by the asm helper. + * clang/LLVM compiles all cases and only discards the unused paths after +-- +2.35.1 +