x86: rename and clean up __copy_from_user_inatomic_nocache()

commit 5de7bcaadf160c1716b20a263cf8f5b06f658959 upstream.

Similarly to the previous commit, this renames the somewhat confusingly
named function.  But in this case, it was at least less confusing: the
__copy_from_user_inatomic_nocache is indeed copying from user memory,
and it is indeed ok to be used in an atomic context, so it will not warn
about it.

But the previous commit also removed the NTB mis-use of the
__copy_from_user_inatomic_nocache() function, and as a result every
call-site is now _actually_ doing a real user copy.  That means that we
can now do the proper user pointer verification too.

End result: add proper address checking, remove the double underscores,
and change the "nocache" to "nontemporal" to more accurately describe
what this x86-only function actually does.  It might be worth noting
that only the target is non-temporal: the actual user accesses are
normal memory accesses.

Also worth noting is that non-x86 targets (and on older 32-bit x86 CPU's
before XMM2 in the Pentium III) we end up just falling back on a regular
user copy, so nothing can actually depend on the non-temporal semantics,
but that has always been true.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 91a3fb8ae7ff4683bdcf13ae8d7f15d5342162c2..269879b9323bd0ab131cc0e003a1be6a1f6e5d84 100644 (file)
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -507,7 +507,7 @@ extern struct movsl_mask {
 } ____cacheline_aligned_in_smp movsl_mask;
 #endif
 
-#define ARCH_HAS_NOCACHE_UACCESS 1
+#define ARCH_HAS_NONTEMPORAL_UACCESS 1
 
 /*
  * The "unsafe" user accesses aren't really "unsafe", but the naming

diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 40379a1adbb84668783310af75345bad12c06a38..fff19e73ccb333344d407bb1a9018c9d63359e00 100644 (file)
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -26,13 +26,7 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n)
        return __copy_user_ll(to, (__force const void *)from, n);
 }
 
-static __always_inline unsigned long
-__copy_from_user_inatomic_nocache(void *to, const void __user *from,
-                                 unsigned long n)
-{
-       return __copy_from_user_ll_nocache_nozero(to, from, n);
-}
-
+unsigned long __must_check copy_from_user_inatomic_nontemporal(void *, const void __user *, unsigned long n);
 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
 

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 6637e7551453dc78d51cb931534dd4d9a72cb06b..52189450b82ea77b1e55b2e48c645f086019a6c8 100644 (file)
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -152,11 +152,12 @@ extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size);
 extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
 
 static inline int
-__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
+copy_from_user_inatomic_nontemporal(void *dst, const void __user *src,
                                  unsigned size)
 {
        long ret;
        kasan_check_write(dst, size);
+       src = mask_user_address(src);
        stac();
        ret = copy_to_nontemporal(dst, (__force const void *)src, size);
        clac();

diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index f6f436f1d57375534b05b948c2c336feffff3c7d..ac27e39fc993d9fcf26ad4080943351ca00c69b3 100644 (file)
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -322,10 +322,11 @@ unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
 }
 EXPORT_SYMBOL(__copy_user_ll);
 
-unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
+unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *from,
                                        unsigned long n)
 {
-       __uaccess_begin_nospec();
+       if (!user_access_begin(from, n))
+               return n;
 #ifdef CONFIG_X86_INTEL_USERCOPY
        if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
                n = __copy_user_intel_nocache(to, from, n);
@@ -334,7 +335,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr
 #else
        __copy_user(to, from, n);
 #endif
-       __uaccess_end();
+       user_access_end();
        return n;
 }
-EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
+EXPORT_SYMBOL(copy_from_user_inatomic_nontemporal);

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index e14a0c3db999b9752af4d5a5b6d1b3b46922493d..f07eeef0f6ec77a787f243cfb0d06dbb07e7512a 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -519,7 +519,7 @@ ggtt_write(struct io_mapping *mapping,
 
        /* We can use the cpu mem copy function because this is X86. */
        vaddr = io_mapping_map_atomic_wc(mapping, base);
-       unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
+       unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset,
                                                      user_data, length);
        io_mapping_unmap_atomic(vaddr);
        if (unwritten) {

diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c
index 506ae1f5e099ff2a9c21989ea175d8369bda6976..cd1901d5c7c0a9c98402ba31f8416ce018ce436b 100644 (file)
--- a/drivers/gpu/drm/qxl/qxl_ioctl.c
+++ b/drivers/gpu/drm/qxl/qxl_ioctl.c
@@ -182,7 +182,7 @@ static int qxl_process_single_command(struct qxl_device *qdev,
 
        /* TODO copy slow path code from i915 */
        fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_MASK));
-       unwritten = __copy_from_user_inatomic_nocache
+       unwritten = copy_from_user_inatomic_nontemporal
                (fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_MASK),
                 u64_to_user_ptr(cmd->command), cmd->command_size);
 

diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 1beb5b395d81d7103f48b53bc8928db739b266cc..7657904c8db9c84f9e9f690983b5360379e6cc02 100644 (file)
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -320,16 +320,21 @@ static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size)
 
 #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
 
-#ifndef ARCH_HAS_NOCACHE_UACCESS
+#ifndef ARCH_HAS_NONTEMPORAL_UACCESS
 
 static inline __must_check unsigned long
-__copy_from_user_inatomic_nocache(void *to, const void __user *from,
+copy_from_user_inatomic_nontemporal(void *to, const void __user *from,
                                  unsigned long n)
 {
+       if (can_do_masked_user_access())
+               from = mask_user_address(from);
+       else
+               if (!access_ok(from, n))
+                       return n;
        return __copy_from_user_inatomic(to, from, n);
 }
 
-#endif         /* ARCH_HAS_NOCACHE_UACCESS */
+#endif         /* ARCH_HAS_NONTEMPORAL_UACCESS */
 
 extern __must_check int check_zeroed_user(const void __user *from, size_t size);
 

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 2fe66a6b8789e7c6c2f262768732810472cb1477..7e81fea459e4671a98f8f916a634e471c97f9a5a 100644 (file)
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -265,7 +265,7 @@ static __always_inline
 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,
                                   size_t len, void *to, void *priv2)
 {
-       return __copy_from_user_inatomic_nocache(to + progress, iter_from, len);
+       return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len);
 }
 
 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)