x86-64: rename misleadingly named '__copy_user_nocache()' function

This function was a masterclass in bad naming, for various historical
reasons.

It claimed to be a non-cached user copy.  It is literally _neither_ of
those things.  It's a specialty memory copy routine that uses
non-temporal stores for the destination (but not the source), and that
does exception handling for both source and destination accesses.

Also note that while it works for unaligned targets, any unaligned parts
(whether at beginning or end) will not use non-temporal stores, since
only words and quadwords can be non-temporal on x86.

The exception handling means that it _can_ be used for user space
accesses, but not on its own - it needs all the normal "start user space
access" logic around it.

But typically the user space access would be the source, not the
non-temporal destination.  That was the original intention of this,
where the destination was some fragile persistent memory target that
needed non-temporal stores in order to catch machine check exceptions
synchronously and deal with them gracefully.

Thus that non-descriptive name: one use case was to copy from user space
into a non-cached kernel buffer.  However, the existing users are a mix
of that intended use-case, and a couple of random drivers that just did
this as a performance tweak.

Some of those random drivers then actively misused the user copying
version (with STAC/CLAC and all) to do kernel copies without ever even
caring about the exception handling, _just_ for the non-temporal
destination.

Rename it as a first small step to actually make it halfway sane, and
change the prototype to be more normal: it doesn't take a user pointer
unless the caller has done the proper conversion, and the argument size
is the full size_t (it still won't actually copy more than 4GB in one
go, but there's also no reason to silently truncate the size argument in
the caller).

Finally, use this now sanely named function in the NTB code, which
mis-used a user copy version (with STAC/CLAC and all) of this interface
despite it not actually being a user copy at all.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 915124011c27648bcb71bb0637cb7f07075abef0..c4b45673e9b80a69de28ed42535d6aebd2578f1d 100644 (file)
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -147,7 +147,8 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
        return copy_user_generic((__force void *)dst, src, size);
 }
 
-extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
+#define copy_to_nontemporal copy_to_nontemporal
+extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size);
 extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
 
 static inline int
@@ -157,7 +158,7 @@ __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
        long ret;
        kasan_check_write(dst, size);
        stac();
-       ret = __copy_user_nocache(dst, src, size);
+       ret = copy_to_nontemporal(dst, (__force const void *)src, size);
        clac();
        return ret;
 }

diff --git a/arch/x86/lib/copy_user_uncached_64.S b/arch/x86/lib/copy_user_uncached_64.S
index 18350b343c2abbaf4731219e7187d3625cdee143..8ed0ce3ad2271f85717f65a88d5d4aad5d78fa6c 100644 (file)
--- a/arch/x86/lib/copy_user_uncached_64.S
+++ b/arch/x86/lib/copy_user_uncached_64.S
@@ -27,7 +27,7 @@
  * Output:
  * rax uncopied bytes or 0 if successful.
  */
-SYM_FUNC_START(__copy_user_nocache)
+SYM_FUNC_START(copy_to_nontemporal)
        ANNOTATE_NOENDBR
        /* If destination is not 7-byte aligned, we'll have to align it */
        testb $7,%dil
@@ -240,5 +240,5 @@ _ASM_EXTABLE_UA(95b, .Ldone)
 _ASM_EXTABLE_UA(52b, .Ldone0)
 _ASM_EXTABLE_UA(53b, .Ldone0)
 
-SYM_FUNC_END(__copy_user_nocache)
-EXPORT_SYMBOL(__copy_user_nocache)
+SYM_FUNC_END(copy_to_nontemporal)
+EXPORT_SYMBOL(copy_to_nontemporal)

diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 654280aaa3e9e05acf0abe435eeb5e4b0bdef463..348c9275edce1421e39277059abfac877fb78bc3 100644 (file)
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -49,11 +49,11 @@ long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
        long rc;
 
        stac();
-       rc = __copy_user_nocache(dst, src, size);
+       rc = copy_to_nontemporal(dst, (__force const void *)src, size);
        clac();
 
        /*
-        * __copy_user_nocache() uses non-temporal stores for the bulk
+        * copy_to_nontemporal() uses non-temporal stores for the bulk
         * of the transfer, but we need to manually flush if the
         * transfer is unaligned. A cached memory copy is used when
         * destination or size is not naturally aligned. That is:

diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 134a79eecfcb876cea25e7ae71c613f37b695948..3467797b5b01d261c3d3ccf1b31d67f6b4a8d91e 100644 (file)
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -92,12 +92,10 @@ static int rvt_wss_llc_size(void)
 static void cacheless_memcpy(void *dst, void *src, size_t n)
 {
        /*
-        * Use the only available X64 cacheless copy.  Add a __user cast
-        * to quiet sparse.  The src agument is already in the kernel so
-        * there are no security issues.  The extra fault recovery machinery
-        * is not invoked.
+        * Use the only available X64 cacheless copy.
+        * The extra fault recovery machinery is not invoked.
         */
-       __copy_user_nocache(dst, (void __user *)src, n);
+       copy_to_nontemporal(dst, src, n);
 }
 
 void rvt_wss_exit(struct rvt_dev_info *rdi)

diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 71d4bb25f7fdd1e46b9b55b68e75fb19e96fd975..21e1d238bc31ef6ff4a9fd725b5e6bc519bbf355 100644 (file)
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -1810,12 +1810,13 @@ static void ntb_tx_copy_callback(void *data,
 
 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset)
 {
-#ifdef ARCH_HAS_NOCACHE_UACCESS
+#ifdef copy_to_nontemporal
        /*
         * Using non-temporal mov to improve performance on non-cached
-        * writes, even though we aren't actually copying from user space.
+        * writes. This only works if __iomem is strictly memory-like,
+        * but that is the case on x86-64
         */
-       __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len);
+       copy_to_nontemporal(offset, entry->buf, entry->len);
 #else
        memcpy_toio(offset, entry->buf, entry->len);
 #endif

diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 3fd98c5b6e1a88c453b4640e6ac53b4ec807a12c..8b15e263371ee84c09adca04a97343fa0730644f 100644 (file)
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -1260,7 +1260,7 @@ static const char *uaccess_safe_builtin[] = {
        "copy_mc_enhanced_fast_string",
        "rep_stos_alternative",
        "rep_movs_alternative",
-       "__copy_user_nocache",
+       "copy_to_nontemporal",
        NULL
 };