x86: support user address masking instead of non-speculative conditional

commit 2865baf54077aa98fcdb478cefe6a42c417b9374 upstream.

The Spectre-v1 mitigations made "access_ok()" much more expensive, since
it has to serialize execution with the test for a valid user address.

All the normal user copy routines avoid this by just masking the user
address with a data-dependent mask instead, but the fast
"unsafe_user_read()" kind of patterms that were supposed to be a fast
case got slowed down.

This introduces a notion of using

	src = masked_user_access_begin(src);

to do the user address sanity using a data-dependent mask instead of the
more traditional conditional

	if (user_read_access_begin(src, len)) {

model.

This model only works for dense accesses that start at 'src' and on
architectures that have a guard region that is guaranteed to fault in
between the user space and the kernel space area.

With this, the user access doesn't need to be manually checked, because
a bad address is guaranteed to fault (by some architecture masking
trick: on x86-64 this involves just turning an invalid user address into
all ones, since we don't map the top of address space).

This only converts a couple of examples for now.  Example x86-64 code
generation for loading two words from user space:

        stac
        mov    %rax,%rcx
        sar    $0x3f,%rcx
        or     %rax,%rcx
        mov    (%rcx),%r13
        mov    0x8(%rcx),%r14
        clac

where all the error handling and -EFAULT is now purely handled out of
line by the exception path.

Of course, if the micro-architecture does badly at 'clac' and 'stac',
the above is still pitifully slow.  But at least we did as well as we
could.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 04789f45ab2b2ffb063045cb00c50b4974dfde49..a10149a96d9e9992fadb4c983427a087161fc99f 100644 (file)
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -53,6 +53,14 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
  */
 #define valid_user_address(x) ((__force long)(x) >= 0)
 
+/*
+ * Masking the user address is an alternative to a conditional
+ * user_access_begin that can avoid the fencing. This only works
+ * for dense accesses starting at the address.
+ */
+#define mask_user_address(x) ((typeof(x))((long)(x)|((long)(x)>>63)))
+#define masked_user_access_begin(x) ({ __uaccess_begin(); mask_user_address(x); })
+
 /*
  * User pointers can have tag bits on x86-64.  This scheme tolerates
  * arbitrary values in those bits rather then masking them off.

diff --git a/fs/select.c b/fs/select.c
index 9515c3fa1a03e8a8576f90f8a4dee46509dd1954..bc185d111436cdc09515a10b145f451416482fcd 100644 (file)
--- a/fs/select.c
+++ b/fs/select.c
@@ -780,7 +780,9 @@ static inline int get_sigset_argpack(struct sigset_argpack *to,
 {
        // the path is hot enough for overhead of copy_from_user() to matter
        if (from) {
-               if (!user_read_access_begin(from, sizeof(*from)))
+               if (can_do_masked_user_access())
+                       from = masked_user_access_begin(from);
+               else if (!user_read_access_begin(from, sizeof(*from)))
                        return -EFAULT;
                unsafe_get_user(to->p, &from->p, Efault);
                unsafe_get_user(to->size, &from->size, Efault);

diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index d8e4105a2f21b1d5bc4ad50b8246a01e483d4b79..39c7cf82b0c22189517f9b10cd528fea121340b2 100644 (file)
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -33,6 +33,13 @@
 })
 #endif
 
+#ifdef masked_user_access_begin
+ #define can_do_masked_user_access() 1
+#else
+ #define can_do_masked_user_access() 0
+ #define masked_user_access_begin(src) NULL
+#endif
+
 /*
  * Architectures should provide two primitives (raw_copy_{to,from}_user())
  * and get rid of their private instances of copy_{to,from}_user() and

diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c
index 6432b8c3e431ecbbb462326bbfc75e63e16c7c44..989a12a678721465c17636b8d98e3962536e7d22 100644 (file)
--- a/lib/strncpy_from_user.c
+++ b/lib/strncpy_from_user.c
@@ -120,6 +120,15 @@ long strncpy_from_user(char *dst, const char __user *src, long count)
        if (unlikely(count <= 0))
                return 0;
 
+       if (can_do_masked_user_access()) {
+               long retval;
+
+               src = masked_user_access_begin(src);
+               retval = do_strncpy_from_user(dst, src, count, count);
+               user_read_access_end();
+               return retval;
+       }
+
        max_addr = TASK_SIZE_MAX;
        src_addr = (unsigned long)untagged_addr(src);
        if (likely(src_addr < max_addr)) {

diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
index feeb935a22991174c7146fbeddc9a002b30dcac0..6e489f9e90f15858ad12858c766bae875a371bda 100644 (file)
--- a/lib/strnlen_user.c
+++ b/lib/strnlen_user.c
@@ -96,6 +96,15 @@ long strnlen_user(const char __user *str, long count)
        if (unlikely(count <= 0))
                return 0;
 
+       if (can_do_masked_user_access()) {
+               long retval;
+
+               str = masked_user_access_begin(str);
+               retval = do_strnlen_user(str, count, count);
+               user_read_access_end();
+               return retval;
+       }
+
        max_addr = TASK_SIZE_MAX;
        src_addr = (unsigned long)untagged_addr(str);
        if (likely(src_addr < max_addr)) {