rdrand: Enable the RDSEED instruction

Enable the use of the RDSEED instruction.  Since RDSEED can more
easily be starved of entropy due to use on other threads, allow it to
fall back to RDRAND by having a function which for every RDSEED
failure executes RDRAND and stores a sample in a separate buffer.

The RDRAND buffer and its high water mark are made static so that they
can accumulate data across multiple invocations.

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

diff --git a/rdrand_asm.S b/rdrand_asm.S
index c92996c17dfe7f5b5c0b8fd7e8cdae686701dc56..9918363d4a34f0e7f12a4e58578bd83a8b677759 100644 (file)
--- a/rdrand_asm.S
+++ b/rdrand_asm.S
@@ -54,6 +54,37 @@ ENTRY(x86_rdrand_bytes)
        jmp     4b
 ENDPROC(x86_rdrand_bytes)
 
+ENTRY(x86_rdseed_or_rdrand_bytes)
+       mov     (%rsi), %r8d            /* RDSEED count */
+       mov     (%rcx), %r9d            /* RDRAND count */
+1:
+       mov     $RDRAND_RETRY_LIMIT, %r10d
+2:
+       .byte   0x48,0x0f,0xc7,0xf8     /* rdseed %rax */
+       jnc     3f
+       mov     %rax, (%rdi)
+       add     $8, %rdi
+       sub     $8, %r8d
+       ja      1b
+4:
+       sub     %r8d, (%rsi)
+       sub     %r9d, (%rcx)
+       ret
+3:
+       .byte   0x48,0x0f,0xc7,0xf0     /* rdrand %rax */
+       jnc     5f
+       mov     %rax, (%rdx)
+       add     $8, %rdx
+       sub     $8, %r9d
+       ja      1b
+       jmp     4b
+5:
+       dec     %r10d
+       rep;nop
+       jnz     2b
+       jmp     4b
+ENDPROC(x86_rdseed_or_rdrand_bytes)
+
 #define SETPTR(var,ptr)        leaq var(%rip),ptr
 #define PTR0   %rdi
 #define PTR1   %rsi
@@ -94,6 +125,55 @@ ENTRY(x86_rdrand_bytes)
        jmp     4b
 ENDPROC(x86_rdrand_bytes)
 
+
+ENTRY(x86_rdseed_or_rdrand_bytes)
+       push    %ebp
+       mov     %esp, %ebp
+       push    %edi
+       push    %esi
+       push    %ebx
+
+       mov     12(%ebp), %ebx
+       mov     20(%ebp), %esi
+       mov     8(%ebp), %edi           /* RDSEED pointer */
+       mov     16(%ebp), %edx          /* RDRAND pointer */
+       mov     (%ebx), %ebx            /* RDSEED count */
+       mov     (%esi), %esi            /* RDRAND count */
+1:
+       mov     $RDRAND_RETRY_LIMIT, %ecx
+2:
+       .byte   0x0f,0xc7,0xf8          /* rdseed %eax */
+       jnc     3f
+       mov     %eax, (%edi)
+       add     $4, %edi
+       sub     $4, %ebx
+       ja      1b
+4:
+       mov     12(%ebp), %edx
+       mov     20(%ebp), %eax
+       sub     %ebx, (%edx)            /* RDSEED count */
+       sub     %esi, (%eax)            /* RDRAND count */
+
+       pop     %ebx
+       pop     %esi
+       pop     %edi
+       pop     %ebp
+       ret
+3:
+       .byte   0x0f,0xc7,0xf0          /* rdrand %eax */
+       jnc     5f
+       mov     %eax, (%edx)
+       add     $4, %edx
+       sub     $4, %esi
+       jnz     1b
+       ja      4b
+5:
+       dec     %ecx
+       rep;nop
+       jnz     2b
+       jmp     4b
+ENDPROC(x86_rdseed_or_rdrand_bytes)
+
 #define SETPTR(var,ptr)        movl $(var),ptr
 #define PTR0   %eax
 #define PTR1   %edx

diff --git a/rngd_rdrand.c b/rngd_rdrand.c
index 3c282824d02e196fc53bc2867fb78a64af8036f3..4d7a5a7b3f52c3af8e677d3c4d758d4d235f1dd6 100644 (file)
--- a/rngd_rdrand.c
+++ b/rngd_rdrand.c
@@ -56,12 +56,20 @@ struct cpuid {
 /*
  * Get data from RDRAND.  The count is in bytes, but the function can
  * round *up* the count to the nearest 4- or 8-byte boundary.  The caller
- * needs to take that into account.
+ * needs to take that into account.  count must not be zero.
  *
  * The function returns the number of bytes actually written.
  */
 extern unsigned int x86_rdrand_bytes(void *ptr, unsigned int count);
 
+/*
+ * Get data from RDSEED (preferentially) or RDRAND into separate
+ * buffers.  Returns when either buffer is full.  Same conditions
+ * apply as for x86_rdrand_bytes().
+ */
+extern void x86_rdseed_or_rdrand_bytes(void *seed_ptr, unsigned int *seed_cnt,
+                                      void *rand_ptr, unsigned int *rand_cnt);
+
 /* Condition RDRAND for seed-grade entropy */
 extern void x86_aes_mangle(void *data, void *state);
 
@@ -129,7 +137,7 @@ static void cpuid(unsigned int leaf, unsigned int subleaf, struct cpuid *out)
 #define RDRAND_ROUNDS          512             /* 512:1 data reduction */
 
 static unsigned char iv_buf[CHUNK_SIZE] __attribute__((aligned(128)));
-static int have_aesni;
+static int have_aesni, have_rdseed;
 
 /* Necessary if we have RDRAND but not AES-NI */
 
@@ -167,41 +175,63 @@ static inline int gcrypt_mangle(unsigned char *tmp)
 
 int xread_drng(void *buf, size_t size, struct rng *ent_src)
 {
+       static unsigned char rdrand_buf[CHUNK_SIZE * RDRAND_ROUNDS]
+               __attribute__((aligned(128)));
+       static unsigned int rdrand_bytes = 0;
+       unsigned char rdseed_buf[CHUNK_SIZE]
+               __attribute__((aligned(128)));
        char *p = buf;
        size_t chunk;
-       void *data;
-       unsigned char rdrand_buf[CHUNK_SIZE * RDRAND_ROUNDS]
-               __attribute__((aligned(128)));
-       unsigned int rand_bytes;
-       int i;
+       unsigned char *rdrand_ptr, *data;
+       unsigned int rand_bytes, seed_bytes;
 
        (void)ent_src;
 
-       rand_bytes = have_aesni
-               ? CHUNK_SIZE * RDRAND_ROUNDS
-               : AES_BLOCK * RDRAND_ROUNDS;
-
        while (size) {
-               if (x86_rdrand_bytes(rdrand_buf, rand_bytes) != rand_bytes) {
-                       message(LOG_DAEMON|LOG_ERR, "read error\n");
-                       return -1;
+               rand_bytes = (have_aesni
+                             ? CHUNK_SIZE * RDRAND_ROUNDS
+                             : AES_BLOCK * RDRAND_ROUNDS)
+                       - rdrand_bytes;
+
+               if (rand_bytes == 0) {
+                       /* We already have a full rdrand_buf */
+                       if (have_aesni) {
+                               x86_aes_mangle(rdrand_buf, iv_buf);
+                               data = iv_buf;
+                               chunk = CHUNK_SIZE;
+                       } else if (!gcrypt_mangle(rdrand_buf)) {
+                               data = rdrand_buf +
+                                       AES_BLOCK * (RDRAND_ROUNDS - 1);
+                               chunk = AES_BLOCK;
+                       } else {
+                               return -1;
+                       }
+                       rdrand_bytes = 0;
+                       goto have_data;
                }
 
-               /*
-                * Use 128-bit AES in CBC mode to reduce the
-                * data by a factor of RDRAND_ROUNDS
-                */
-               if (have_aesni) {
-                       x86_aes_mangle(rdrand_buf, iv_buf);
-                       data = iv_buf;
-                       chunk = CHUNK_SIZE;
-               } else if (!gcrypt_mangle(rdrand_buf)) {
-                       data = rdrand_buf + AES_BLOCK * (RDRAND_ROUNDS - 1);
-                       chunk = AES_BLOCK;
+               rdrand_ptr = rdrand_buf + rdrand_bytes;
+
+               if (have_rdseed) {
+                       seed_bytes = sizeof rdseed_buf;
+                       x86_rdseed_or_rdrand_bytes(rdseed_buf, &seed_bytes,
+                                                  rdrand_ptr, &rand_bytes);
                } else {
-                       return -1;
+                       rand_bytes = x86_rdrand_bytes(rdrand_ptr, rand_bytes);
+                       seed_bytes = 0;
                }
 
+               rdrand_bytes += rand_bytes;
+
+               if (seed_bytes) {
+                       data = rdseed_buf;
+                       chunk = seed_bytes;
+                       goto have_data;
+               }
+
+               continue;       /* No data ready yet */
+
+       have_data:
                chunk = (chunk > size) ? size : chunk;
                memcpy(p, data, chunk);
                p += chunk;
@@ -269,6 +299,8 @@ int init_drng_entropy_source(struct rng *ent_src)
        /* We need RDRAND, but AESni is optional */
        const uint32_t features_ecx1_rdrand = 1 << 30;
        const uint32_t features_ecx1_aesni  = 1 << 25;
+       const uint32_t features_ebx7_rdseed = 1 << 18;
+       uint32_t max_cpuid_leaf;
        static unsigned char key[AES_BLOCK] = {
                0x00,0x10,0x20,0x30,0x40,0x50,0x60,0x70,
                0x80,0x90,0xa0,0xb0,0xc0,0xd0,0xe0,0xf0
@@ -281,14 +313,24 @@ int init_drng_entropy_source(struct rng *ent_src)
                return 1;       /* No CPUID instruction */
 
        cpuid(0, 0, &info);
-       if (info.eax < 1)
+       max_cpuid_leaf = info.eax;
+
+       if (max_cpuid_leaf < 1)
                return 1;
+
        cpuid(1, 0, &info);
        if (!(info.ecx & features_ecx1_rdrand))
                return 1;
 
        have_aesni = !!(info.ecx & features_ecx1_aesni);
 
+       have_rdseed = 0;
+       if (max_cpuid_leaf >= 7) {
+               cpuid(7, 0, &info);
+               if (info.ebx & features_ebx7_rdseed)
+                       have_rdseed = 1;
+       }
+
        /* Randomize the AES data reduction key the best we can */
        if (x86_rdrand_bytes(xkey, sizeof xkey) != sizeof xkey)
                return 1;
@@ -299,7 +341,7 @@ int init_drng_entropy_source(struct rng *ent_src)
                close(fd);
        }
 
-       for (i = 0; i < sizeof key; i++)
+       for (i = 0; i < (int)sizeof key; i++)
                key[i] ^= xkey[i];
 
        /* Initialize the IV buffer */