rdrand: Fix the RDRAND data reduction

The RDRAND data reduction function was not correct.  Specifically:

1. When using AESni, in order to guarantee at least one reseed event
   per reduction stripe we need to process the data in a different
   order.  This means writing it out all the data to a buffer before
   processing it, and then process it in much larger stripes.

2. When using gcrypt, we are only performing one reduction at a time,
   so only process enough input for one reduction and only generate
   that amount of output.

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

diff --git a/rdrand_asm.S b/rdrand_asm.S
index 913b923b445b4efb16ed8bd8bd2771629ef43497..c92996c17dfe7f5b5c0b8fd7e8cdae686701dc56 100644 (file)
--- a/rdrand_asm.S
+++ b/rdrand_asm.S
@@ -58,6 +58,7 @@ ENDPROC(x86_rdrand_bytes)
 #define PTR0   %rdi
 #define PTR1   %rsi
 #define PTR2   %rcx
+#define CTR3   %eax
 #define NPTR2  1       /* %rcx = %r1, only 0-7 valid here */
 
 #elif defined(__i386__)
@@ -97,6 +98,7 @@ ENDPROC(x86_rdrand_bytes)
 #define PTR0   %eax
 #define PTR1   %edx
 #define PTR2   %ecx
+#define CTR3   %esi
 #define NPTR2  1       /* %rcx = %r1 */
 
 #endif
@@ -107,27 +109,37 @@ ENTRY(x86_aes_mangle)
        mov     %esp, %ebp
        movl    8(%ebp), %eax
        movl    12(%ebp), %edx
+       push    %esi
 #endif
+       movl    $512, CTR3      /* Number of rounds */
+       
+       movdqa  (0*16)(PTR1), %xmm0
+       movdqa  (1*16)(PTR1), %xmm1
+       movdqa  (2*16)(PTR1), %xmm2
+       movdqa  (3*16)(PTR1), %xmm3
+       movdqa  (4*16)(PTR1), %xmm4
+       movdqa  (5*16)(PTR1), %xmm5
+       movdqa  (6*16)(PTR1), %xmm6
+       movdqa  (7*16)(PTR1), %xmm7
 
+#ifdef __x86_64__
        SETPTR(aes_round_keys, PTR2)
+1:
+#else
+1:
+       SETPTR(aes_round_keys, PTR2)
+#endif
 
-       movdqa  (0*16)(PTR0), %xmm0
-       movdqa  (1*16)(PTR0), %xmm1
-       movdqa  (2*16)(PTR0), %xmm2
-       movdqa  (3*16)(PTR0), %xmm3
-       movdqa  (4*16)(PTR0), %xmm4
-       movdqa  (5*16)(PTR0), %xmm5
-       movdqa  (6*16)(PTR0), %xmm6
-       movdqa  (7*16)(PTR0), %xmm7
-
-       pxor    (0*16)(PTR1), %xmm0
-       pxor    (1*16)(PTR1), %xmm1
-       pxor    (2*16)(PTR1), %xmm2
-       pxor    (3*16)(PTR1), %xmm3
-       pxor    (4*16)(PTR1), %xmm4
-       pxor    (5*16)(PTR1), %xmm5
-       pxor    (6*16)(PTR1), %xmm6
-       pxor    (7*16)(PTR1), %xmm7
+       /* 8192 = 512 (rounds) * 16 (bytes) */
+       pxor    (0*8192)(PTR0), %xmm0
+       pxor    (1*8192)(PTR0), %xmm1
+       pxor    (2*8192)(PTR0), %xmm2
+       pxor    (3*8192)(PTR0), %xmm3
+       pxor    (4*8192)(PTR0), %xmm4
+       pxor    (5*8192)(PTR0), %xmm5
+       pxor    (6*8192)(PTR0), %xmm6
+       pxor    (7*8192)(PTR0), %xmm7
+       add     $16, PTR0
 
 offset = 0
        .rept 10
@@ -175,16 +187,9 @@ offset = offset + 16
        .byte   0x66,0x0f,0x38,0xdd,0x30+NPTR2  /* aesenclast (PTR2), %xmm6 */
        .byte   0x66,0x0f,0x38,0xdd,0x38+NPTR2  /* aesenclast (PTR2), %xmm7 */
 #endif
-
-       movdqa  %xmm0, (0*16)(PTR0)
-       movdqa  %xmm1, (1*16)(PTR0)
-       movdqa  %xmm2, (2*16)(PTR0)
-       movdqa  %xmm3, (3*16)(PTR0)
-       movdqa  %xmm4, (4*16)(PTR0)
-       movdqa  %xmm5, (5*16)(PTR0)
-       movdqa  %xmm6, (6*16)(PTR0)
-       movdqa  %xmm7, (7*16)(PTR0)
-
+       sub     $1, CTR3
+       jnz     1b
+       
        movdqa  %xmm0, (0*16)(PTR1)
        movdqa  %xmm1, (1*16)(PTR1)
        movdqa  %xmm2, (2*16)(PTR1)
@@ -195,6 +200,7 @@ offset = offset + 16
        movdqa  %xmm7, (7*16)(PTR1)
 
 #ifdef __i386__
+       pop     %esi
        pop     %ebp
 #endif
        ret

diff --git a/rngd_rdrand.c b/rngd_rdrand.c
index cb6736984953c9105753953d82683f829734c50e..3c282824d02e196fc53bc2867fb78a64af8036f3 100644 (file)
--- a/rngd_rdrand.c
+++ b/rngd_rdrand.c
@@ -124,7 +124,9 @@ static void cpuid(unsigned int leaf, unsigned int subleaf, struct cpuid *out)
 }
 
 /* Read data from the drng in chunks of 128 bytes for AES scrambling */
-#define CHUNK_SIZE             (16*8)
+#define AES_BLOCK              16
+#define CHUNK_SIZE             (AES_BLOCK*8)   /* 8 parallel streams */
+#define RDRAND_ROUNDS          512             /* 512:1 data reduction */
 
 static unsigned char iv_buf[CHUNK_SIZE] __attribute__((aligned(128)));
 static int have_aesni;
@@ -146,7 +148,8 @@ static inline int gcrypt_mangle(unsigned char *tmp)
 
        /* Encrypt tmp in-place. */
 
-       gcry_error = gcry_cipher_encrypt(gcry_cipher_hd, tmp, CHUNK_SIZE,
+       gcry_error = gcry_cipher_encrypt(gcry_cipher_hd, tmp,
+                                        AES_BLOCK * RDRAND_ROUNDS,
                                         NULL, 0);
 
        if (gcry_error) {
@@ -166,30 +169,41 @@ int xread_drng(void *buf, size_t size, struct rng *ent_src)
 {
        char *p = buf;
        size_t chunk;
-       const int rdrand_round_count = 512;
-       unsigned char tmp[CHUNK_SIZE] __attribute__((aligned(128)));
+       void *data;
+       unsigned char rdrand_buf[CHUNK_SIZE * RDRAND_ROUNDS]
+               __attribute__((aligned(128)));
+       unsigned int rand_bytes;
        int i;
 
        (void)ent_src;
 
+       rand_bytes = have_aesni
+               ? CHUNK_SIZE * RDRAND_ROUNDS
+               : AES_BLOCK * RDRAND_ROUNDS;
+
        while (size) {
-               for (i = 0; i < rdrand_round_count; i++) {
-                       if (x86_rdrand_bytes(tmp, CHUNK_SIZE) != CHUNK_SIZE) {
-                               message(LOG_DAEMON|LOG_ERR, "read error\n");
-                               return -1;
-                       }
-
-                       /*
-                        * Use 128-bit AES in CBC mode to reduce the
-                        * data by a factor of rdrand_round_count
-                        */
-                       if (have_aesni)
-                               x86_aes_mangle(tmp, iv_buf);
-                       else if (gcrypt_mangle(tmp))
-                               return -1;
+               if (x86_rdrand_bytes(rdrand_buf, rand_bytes) != rand_bytes) {
+                       message(LOG_DAEMON|LOG_ERR, "read error\n");
+                       return -1;
+               }
+
+               /*
+                * Use 128-bit AES in CBC mode to reduce the
+                * data by a factor of RDRAND_ROUNDS
+                */
+               if (have_aesni) {
+                       x86_aes_mangle(rdrand_buf, iv_buf);
+                       data = iv_buf;
+                       chunk = CHUNK_SIZE;
+               } else if (!gcrypt_mangle(rdrand_buf)) {
+                       data = rdrand_buf + AES_BLOCK * (RDRAND_ROUNDS - 1);
+                       chunk = AES_BLOCK;
+               } else {
+                       return -1;
                }
-               chunk = (sizeof(tmp) > size) ? size : sizeof(tmp);
-               memcpy(p, tmp, chunk);
+
+               chunk = (chunk > size) ? size : chunk;
+               memcpy(p, data, chunk);
                p += chunk;
                size -= chunk;
        }
@@ -222,14 +236,14 @@ static int init_gcrypt(const void *key)
                                      GCRY_CIPHER_MODE_CBC, 0);
 
        if (!gcry_error)
-               gcry_error = gcry_cipher_setkey(gcry_cipher_hd, key, 16);
+               gcry_error = gcry_cipher_setkey(gcry_cipher_hd, key, AES_BLOCK);
 
        if (!gcry_error) {
                /*
                 * Only need the first 16 bytes of iv_buf. AES-NI can
                 * encrypt multiple blocks in parallel but we can't.
                 */
-               gcry_error = gcry_cipher_setiv(gcry_cipher_hd, iv_buf, 16);
+               gcry_error = gcry_cipher_setiv(gcry_cipher_hd, iv_buf, AES_BLOCK);
        }
 
        if (gcry_error) {
@@ -255,11 +269,11 @@ int init_drng_entropy_source(struct rng *ent_src)
        /* We need RDRAND, but AESni is optional */
        const uint32_t features_ecx1_rdrand = 1 << 30;
        const uint32_t features_ecx1_aesni  = 1 << 25;
-       static unsigned char key[16] = {
+       static unsigned char key[AES_BLOCK] = {
                0x00,0x10,0x20,0x30,0x40,0x50,0x60,0x70,
                0x80,0x90,0xa0,0xb0,0xc0,0xd0,0xe0,0xf0
        }; /* AES data reduction key */
-       unsigned char xkey[16]; /* Material to XOR into the key */
+       unsigned char xkey[AES_BLOCK];  /* Material to XOR into the key */
        int fd;
        int i;