x86: rename and clean up __copy_from_user_inatomic_nocache()

Similarly to the previous commit, this renames the somewhat confusingly
named function.  But in this case, it was at least less confusing: the
__copy_from_user_inatomic_nocache is indeed copying from user memory,
and it is indeed ok to be used in an atomic context, so it will not warn
about it.

But the previous commit also removed the NTB mis-use of the
__copy_from_user_inatomic_nocache() function, and as a result every
call-site is now _actually_ doing a real user copy.  That means that we
can now do the proper user pointer verification too.

End result: add proper address checking, remove the double underscores,
and change the "nocache" to "nontemporal" to more accurately describe
what this x86-only function actually does.  It might be worth noting
that only the target is non-temporal: the actual user accesses are
normal memory accesses.

Also worth noting is that non-x86 targets (and on older 32-bit x86 CPU's
before XMM2 in the Pentium III) we end up just falling back on a regular
user copy, so nothing can actually depend on the non-temporal semantics,
but that has always been true.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 367297b188c39cfc0ae6d52a06a5c5e6f170a09b..3a0dd3c2b233e4dbcf134d3ad0e469a3dcc2f09e 100644 (file)
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -507,7 +507,7 @@ extern struct movsl_mask {
 } ____cacheline_aligned_in_smp movsl_mask;
 #endif
 
-#define ARCH_HAS_NOCACHE_UACCESS 1
+#define ARCH_HAS_NONTEMPORAL_UACCESS 1
 
 /*
  * The "unsafe" user accesses aren't really "unsafe", but the naming

diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 40379a1adbb84668783310af75345bad12c06a38..fff19e73ccb333344d407bb1a9018c9d63359e00 100644 (file)
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -26,13 +26,7 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n)
        return __copy_user_ll(to, (__force const void *)from, n);
 }
 
-static __always_inline unsigned long
-__copy_from_user_inatomic_nocache(void *to, const void __user *from,
-                                 unsigned long n)
-{
-       return __copy_from_user_ll_nocache_nozero(to, from, n);
-}
-
+unsigned long __must_check copy_from_user_inatomic_nontemporal(void *, const void __user *, unsigned long n);
 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
 

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index c4b45673e9b80a69de28ed42535d6aebd2578f1d..859d99c31c29ab712841ed9ceb4a86fd175b6cb7 100644 (file)
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -152,11 +152,12 @@ extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size);
 extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
 
 static inline int
-__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
+copy_from_user_inatomic_nontemporal(void *dst, const void __user *src,
                                  unsigned size)
 {
        long ret;
        kasan_check_write(dst, size);
+       src = mask_user_address(src);
        stac();
        ret = copy_to_nontemporal(dst, (__force const void *)src, size);
        clac();

diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index f6f436f1d57375534b05b948c2c336feffff3c7d..ac27e39fc993d9fcf26ad4080943351ca00c69b3 100644 (file)
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -322,10 +322,11 @@ unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
 }
 EXPORT_SYMBOL(__copy_user_ll);
 
-unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
+unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *from,
                                        unsigned long n)
 {
-       __uaccess_begin_nospec();
+       if (!user_access_begin(from, n))
+               return n;
 #ifdef CONFIG_X86_INTEL_USERCOPY
        if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
                n = __copy_user_intel_nocache(to, from, n);
@@ -334,7 +335,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr
 #else
        __copy_user(to, from, n);
 #endif
-       __uaccess_end();
+       user_access_end();
        return n;
 }
-EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
+EXPORT_SYMBOL(copy_from_user_inatomic_nontemporal);

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 4c82c9544b932840bfb6f13a254438b5fa27e9b2..72fe91ed1c74033ba1e31486bc1450b2ee06c115 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -520,7 +520,7 @@ ggtt_write(struct io_mapping *mapping,
 
        /* We can use the cpu mem copy function because this is X86. */
        vaddr = io_mapping_map_atomic_wc(mapping, base);
-       unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
+       unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset,
                                                      user_data, length);
        io_mapping_unmap_atomic(vaddr);
        if (unwritten) {

diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c
index 336cbff2608915189c12fdb57d54635b8096e444..26545a08cdf771b96e479a692e1e68135df360e5 100644 (file)
--- a/drivers/gpu/drm/qxl/qxl_ioctl.c
+++ b/drivers/gpu/drm/qxl/qxl_ioctl.c
@@ -184,7 +184,7 @@ static int qxl_process_single_command(struct qxl_device *qdev,
 
        /* TODO copy slow path code from i915 */
        fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_MASK));
-       unwritten = __copy_from_user_inatomic_nocache
+       unwritten = copy_from_user_inatomic_nontemporal
                (fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_MASK),
                 u64_to_user_ptr(cmd->command), cmd->command_size);
 

diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 1f3804245c06679d4c124f350352d8dc37d56219..4c7d0b815093d2326c1ea5d5296465e3fe57827a 100644 (file)
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -331,16 +331,21 @@ static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size)
 
 #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
 
-#ifndef ARCH_HAS_NOCACHE_UACCESS
+#ifndef ARCH_HAS_NONTEMPORAL_UACCESS
 
 static inline __must_check unsigned long
-__copy_from_user_inatomic_nocache(void *to, const void __user *from,
+copy_from_user_inatomic_nontemporal(void *to, const void __user *from,
                                  unsigned long n)
 {
+       if (can_do_masked_user_access())
+               from = mask_user_address(from);
+       else
+               if (!access_ok(from, n))
+                       return n;
        return __copy_from_user_inatomic(to, from, n);
 }
 
-#endif         /* ARCH_HAS_NOCACHE_UACCESS */
+#endif         /* ARCH_HAS_NONTEMPORAL_UACCESS */
 
 extern __must_check int check_zeroed_user(const void __user *from, size_t size);
 

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 896760bad455fc535fce20159e6d33bf3cd7a7b5..b3a7642ced4ff51707cf20ca4b3a91f18dd03d1d 100644 (file)
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -277,7 +277,7 @@ static __always_inline
 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,
                                   size_t len, void *to, void *priv2)
 {
-       return __copy_from_user_inatomic_nocache(to + progress, iter_from, len);
+       return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len);
 }
 
 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)