x86-64: rename misleadingly named '__copy_user_nocache()' function

commit d187a86de793f84766ea40b9ade7ac60aabbb4fe upstream.

This function was a masterclass in bad naming, for various historical
reasons.

It claimed to be a non-cached user copy.  It is literally _neither_ of
those things.  It's a specialty memory copy routine that uses
non-temporal stores for the destination (but not the source), and that
does exception handling for both source and destination accesses.

Also note that while it works for unaligned targets, any unaligned parts
(whether at beginning or end) will not use non-temporal stores, since
only words and quadwords can be non-temporal on x86.

The exception handling means that it _can_ be used for user space
accesses, but not on its own - it needs all the normal "start user space
access" logic around it.

But typically the user space access would be the source, not the
non-temporal destination.  That was the original intention of this,
where the destination was some fragile persistent memory target that
needed non-temporal stores in order to catch machine check exceptions
synchronously and deal with them gracefully.

Thus that non-descriptive name: one use case was to copy from user space
into a non-cached kernel buffer.  However, the existing users are a mix
of that intended use-case, and a couple of random drivers that just did
this as a performance tweak.

Some of those random drivers then actively misused the user copying
version (with STAC/CLAC and all) to do kernel copies without ever even
caring about the exception handling, _just_ for the non-temporal
destination.

Rename it as a first small step to actually make it halfway sane, and
change the prototype to be more normal: it doesn't take a user pointer
unless the caller has done the proper conversion, and the argument size
is the full size_t (it still won't actually copy more than 4GB in one
go, but there's also no reason to silently truncate the size argument in
the caller).

Finally, use this now sanely named function in the NTB code, which
mis-used a user copy version (with STAC/CLAC and all) of this interface
despite it not actually being a user copy at all.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 915124011c27648bcb71bb0637cb7f07075abef0..c4b45673e9b80a69de28ed42535d6aebd2578f1d 100644 (file)
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -147,7 +147,8 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
        return copy_user_generic((__force void *)dst, src, size);
 }
 
-extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
+#define copy_to_nontemporal copy_to_nontemporal
+extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size);
 extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
 
 static inline int
@@ -157,7 +158,7 @@ __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
        long ret;
        kasan_check_write(dst, size);
        stac();
-       ret = __copy_user_nocache(dst, src, size);
+       ret = copy_to_nontemporal(dst, (__force const void *)src, size);
        clac();
        return ret;
 }

diff --git a/arch/x86/lib/copy_user_uncached_64.S b/arch/x86/lib/copy_user_uncached_64.S
index 18350b343c2abbaf4731219e7187d3625cdee143..8ed0ce3ad2271f85717f65a88d5d4aad5d78fa6c 100644 (file)
--- a/arch/x86/lib/copy_user_uncached_64.S
+++ b/arch/x86/lib/copy_user_uncached_64.S
@@ -27,7 +27,7 @@
  * Output:
  * rax uncopied bytes or 0 if successful.
  */
-SYM_FUNC_START(__copy_user_nocache)
+SYM_FUNC_START(copy_to_nontemporal)
        ANNOTATE_NOENDBR
        /* If destination is not 7-byte aligned, we'll have to align it */
        testb $7,%dil
@@ -240,5 +240,5 @@ _ASM_EXTABLE_UA(95b, .Ldone)
 _ASM_EXTABLE_UA(52b, .Ldone0)
 _ASM_EXTABLE_UA(53b, .Ldone0)
 
-SYM_FUNC_END(__copy_user_nocache)
-EXPORT_SYMBOL(__copy_user_nocache)
+SYM_FUNC_END(copy_to_nontemporal)
+EXPORT_SYMBOL(copy_to_nontemporal)

diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 654280aaa3e9e05acf0abe435eeb5e4b0bdef463..348c9275edce1421e39277059abfac877fb78bc3 100644 (file)
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -49,11 +49,11 @@ long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
        long rc;
 
        stac();
-       rc = __copy_user_nocache(dst, src, size);
+       rc = copy_to_nontemporal(dst, (__force const void *)src, size);
        clac();
 
        /*
-        * __copy_user_nocache() uses non-temporal stores for the bulk
+        * copy_to_nontemporal() uses non-temporal stores for the bulk
         * of the transfer, but we need to manually flush if the
         * transfer is unaligned. A cached memory copy is used when
         * destination or size is not naturally aligned. That is:

diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index c1199ea5d41fda9073bc7381bc54c5d33e0887dc..3c7ee7ddc5dd7891db20629d44f04f69f6718a9f 100644 (file)
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -92,12 +92,10 @@ static int rvt_wss_llc_size(void)
 static void cacheless_memcpy(void *dst, void *src, size_t n)
 {
        /*
-        * Use the only available X64 cacheless copy.  Add a __user cast
-        * to quiet sparse.  The src agument is already in the kernel so
-        * there are no security issues.  The extra fault recovery machinery
-        * is not invoked.
+        * Use the only available X64 cacheless copy.
+        * The extra fault recovery machinery is not invoked.
         */
-       __copy_user_nocache(dst, (void __user *)src, n);
+       copy_to_nontemporal(dst, src, n);
 }
 
 void rvt_wss_exit(struct rvt_dev_info *rdi)

diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 78e02fe6caba0cabe6a94871a7ae1cd5beedd6fa..2e77b699be2a40c47d12e89b4534b5508f662bc9 100644 (file)
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -1779,12 +1779,13 @@ static void ntb_tx_copy_callback(void *data,
 
 static void ntb_memcpy_tx_on_stack(struct ntb_queue_entry *entry, void __iomem *offset)
 {
-#ifdef ARCH_HAS_NOCACHE_UACCESS
+#ifdef copy_to_nontemporal
        /*
         * Using non-temporal mov to improve performance on non-cached
-        * writes, even though we aren't actually copying from user space.
+        * writes. This only works if __iomem is strictly memory-like,
+        * but that is the case on x86-64
         */
-       __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len);
+       copy_to_nontemporal(offset, entry->buf, entry->len);
 #else
        memcpy_toio(offset, entry->buf, entry->len);
 #endif

diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index b6765e8765074d54b118e4731b6879cb34df26e4..731fd595ac45b9e743e88344bfbffc5f09815570 100644 (file)
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -1301,7 +1301,7 @@ static const char *uaccess_safe_builtin[] = {
        "copy_mc_enhanced_fast_string",
        "rep_stos_alternative",
        "rep_movs_alternative",
-       "__copy_user_nocache",
+       "copy_to_nontemporal",
        NULL
 };