fat-x86_64.c

   1 /* fat-x86_64.c
   2
   3    Copyright (C) 2015 Niels Möller
   4
   5    This file is part of GNU Nettle.
   6
   7    GNU Nettle is free software: you can redistribute it and/or
   8    modify it under the terms of either:
   9
  10      * the GNU Lesser General Public License as published by the Free
  11        Software Foundation; either version 3 of the License, or (at your
  12        option) any later version.
  13
  14    or
  15
  16      * the GNU General Public License as published by the Free
  17        Software Foundation; either version 2 of the License, or (at your
  18        option) any later version.
  19
  20    or both in parallel, as here.
  21
  22    GNU Nettle is distributed in the hope that it will be useful,
  23    but WITHOUT ANY WARRANTY; without even the implied warranty of
  24    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  25    General Public License for more details.
  26
  27    You should have received copies of the GNU General Public License and
  28    the GNU Lesser General Public License along with this program.  If
  29    not, see http://www.gnu.org/licenses/.
  30 */
  31
  32 #define _GNU_SOURCE
  33
  34 #if HAVE_CONFIG_H
  35 # include "config.h"
  36 #endif
  37
  38 #include <assert.h>
  39 #include <stdio.h>
  40 #include <stdlib.h>
  41 #include <string.h>
  42
  43 #include "nettle-types.h"
  44
  45 #include "aes-internal.h"
  46 #include "ghash-internal.h"
  47 #include "memxor.h"
  48 #include "fat-setup.h"
  49
  50 void _nettle_cpuid (uint32_t input, uint32_t regs[4]);
  51
  52 struct x86_features
  53 {
  54   enum x86_vendor { X86_OTHER, X86_INTEL, X86_AMD } vendor;
  55   int have_aesni;
  56   int have_sha_ni;
  57   int have_pclmul;
  58 };
  59
  60 #define SKIP(s, slen, literal, llen)                            \
  61   (((slen) >= (llen) && memcmp ((s), (literal), llen) == 0)     \
  62    ? ((slen) -= (llen), (s) += (llen), 1) : 0)
  63 #define MATCH(s, slen, literal, llen)                           \
  64   ((slen) == (llen) && memcmp ((s), (literal), llen) == 0)
  65
  66 static void
  67 get_x86_features (struct x86_features *features)
  68 {
  69   const char *s;
  70   features->vendor = X86_OTHER;
  71   features->have_aesni = 0;
  72   features->have_sha_ni = 0;
  73   features->have_pclmul = 0;
  74
  75   s = secure_getenv (ENV_OVERRIDE);
  76   if (s)
  77     for (;;)
  78       {
  79         const char *sep = strchr (s, ',');
  80         size_t length = sep ? (size_t) (sep - s) : strlen(s);
  81
  82         if (SKIP (s, length, "vendor:", 7))
  83           {
  84             if (MATCH(s, length, "intel", 5))
  85               features->vendor = X86_INTEL;
  86             else if (MATCH(s, length, "amd", 3))
  87               features->vendor = X86_AMD;
  88
  89           }
  90         else if (MATCH (s, length, "aesni", 5))
  91           features->have_aesni = 1;
  92         else if (MATCH (s, length, "sha_ni", 6))
  93           features->have_sha_ni = 1;
  94         else if (MATCH (s, length, "pclmul", 6))
  95           features->have_pclmul = 1;
  96         if (!sep)
  97           break;
  98         s = sep + 1;
  99       }
 100   else
 101     {
 102       uint32_t cpuid_data[4];
 103       _nettle_cpuid (0, cpuid_data);
 104       if (memcmp (cpuid_data + 1, "Genu" "ntel" "ineI", 12) == 0)
 105         features->vendor = X86_INTEL;
 106       else if (memcmp (cpuid_data + 1, "Auth" "cAMD" "enti", 12) == 0)
 107         features->vendor = X86_AMD;
 108
 109       _nettle_cpuid (1, cpuid_data);
 110       if (cpuid_data[2] & 0x2)
 111         features->have_pclmul = 1;
 112       if (cpuid_data[2] & 0x02000000)
 113         features->have_aesni = 1;
 114
 115       _nettle_cpuid (7, cpuid_data);
 116       if (cpuid_data[1] & 0x20000000)
 117         features->have_sha_ni = 1;
 118     }
 119 }
 120
 121 DECLARE_FAT_FUNC(nettle_aes128_encrypt, aes128_crypt_func)
 122 DECLARE_FAT_FUNC(nettle_aes128_decrypt, aes128_crypt_func)
 123 DECLARE_FAT_FUNC_VAR(aes128_encrypt, aes128_crypt_func, c)
 124 DECLARE_FAT_FUNC_VAR(aes128_encrypt, aes128_crypt_func, aesni)
 125 DECLARE_FAT_FUNC_VAR(aes128_decrypt, aes128_crypt_func, c)
 126 DECLARE_FAT_FUNC_VAR(aes128_decrypt, aes128_crypt_func, aesni)
 127 DECLARE_FAT_FUNC(nettle_aes192_encrypt, aes192_crypt_func)
 128 DECLARE_FAT_FUNC(nettle_aes192_decrypt, aes192_crypt_func)
 129 DECLARE_FAT_FUNC_VAR(aes192_encrypt, aes192_crypt_func, c)
 130 DECLARE_FAT_FUNC_VAR(aes192_encrypt, aes192_crypt_func, aesni)
 131 DECLARE_FAT_FUNC_VAR(aes192_decrypt, aes192_crypt_func, c)
 132 DECLARE_FAT_FUNC_VAR(aes192_decrypt, aes192_crypt_func, aesni)
 133 DECLARE_FAT_FUNC(nettle_aes256_encrypt, aes256_crypt_func)
 134 DECLARE_FAT_FUNC(nettle_aes256_decrypt, aes256_crypt_func)
 135 DECLARE_FAT_FUNC_VAR(aes256_encrypt, aes256_crypt_func, c)
 136 DECLARE_FAT_FUNC_VAR(aes256_encrypt, aes256_crypt_func, aesni)
 137 DECLARE_FAT_FUNC_VAR(aes256_decrypt, aes256_crypt_func, c)
 138 DECLARE_FAT_FUNC_VAR(aes256_decrypt, aes256_crypt_func, aesni)
 139
 140 DECLARE_FAT_FUNC(nettle_cbc_aes128_encrypt, cbc_aes128_encrypt_func);
 141 DECLARE_FAT_FUNC_VAR(cbc_aes128_encrypt, cbc_aes128_encrypt_func, c);
 142 DECLARE_FAT_FUNC_VAR(cbc_aes128_encrypt, cbc_aes128_encrypt_func, aesni);
 143 DECLARE_FAT_FUNC(nettle_cbc_aes192_encrypt, cbc_aes192_encrypt_func);
 144 DECLARE_FAT_FUNC_VAR(cbc_aes192_encrypt, cbc_aes192_encrypt_func, c);
 145 DECLARE_FAT_FUNC_VAR(cbc_aes192_encrypt, cbc_aes192_encrypt_func, aesni);
 146 DECLARE_FAT_FUNC(nettle_cbc_aes256_encrypt, cbc_aes256_encrypt_func);
 147 DECLARE_FAT_FUNC_VAR(cbc_aes256_encrypt, cbc_aes256_encrypt_func, c);
 148 DECLARE_FAT_FUNC_VAR(cbc_aes256_encrypt, cbc_aes256_encrypt_func, aesni);
 149
 150 DECLARE_FAT_FUNC(nettle_memxor, memxor_func)
 151 DECLARE_FAT_FUNC_VAR(memxor, memxor_func, x86_64)
 152 DECLARE_FAT_FUNC_VAR(memxor, memxor_func, sse2)
 153
 154 DECLARE_FAT_FUNC(nettle_sha1_compress, sha1_compress_func)
 155 DECLARE_FAT_FUNC_VAR(sha1_compress, sha1_compress_func, x86_64)
 156 DECLARE_FAT_FUNC_VAR(sha1_compress, sha1_compress_func, sha_ni)
 157
 158 DECLARE_FAT_FUNC(_nettle_sha256_compress_n, sha256_compress_n_func)
 159 DECLARE_FAT_FUNC_VAR(sha256_compress_n, sha256_compress_n_func, x86_64)
 160 DECLARE_FAT_FUNC_VAR(sha256_compress_n, sha256_compress_n_func, sha_ni)
 161
 162 DECLARE_FAT_FUNC(_nettle_ghash_set_key, ghash_set_key_func)
 163 DECLARE_FAT_FUNC_VAR(ghash_set_key, ghash_set_key_func, c)
 164 DECLARE_FAT_FUNC_VAR(ghash_set_key, ghash_set_key_func, pclmul)
 165
 166 DECLARE_FAT_FUNC(_nettle_ghash_update, ghash_update_func)
 167 DECLARE_FAT_FUNC_VAR(ghash_update, ghash_update_func, table)
 168 DECLARE_FAT_FUNC_VAR(ghash_update, ghash_update_func, pclmul)
 169
 170
 171 /* This function should usually be called only once, at startup. But
 172    it is idempotent, and on x86, pointer updates are atomic, so
 173    there's no danger if it is called simultaneously from multiple
 174    threads. */
 175 static void CONSTRUCTOR
 176 fat_init (void)
 177 {
 178   struct x86_features features;
 179   int verbose;
 180
 181   /* FIXME: Replace all getenv calls by getenv_secure? */
 182   verbose = getenv (ENV_VERBOSE) != NULL;
 183   if (verbose)
 184     fprintf (stderr, "libnettle: fat library initialization.\n");
 185
 186   get_x86_features (&features);
 187   if (verbose)
 188     {
 189       const char * const vendor_names[3] =
 190         { "other", "intel", "amd" };
 191       fprintf (stderr, "libnettle: cpu features: vendor:%s%s%s%s\n",
 192                vendor_names[features.vendor],
 193                features.have_aesni ? ",aesni" : "",
 194                features.have_sha_ni ? ",sha_ni" : "",
 195                features.have_pclmul ? ",pclmul" : "");
 196     }
 197   if (features.have_aesni)
 198     {
 199       if (verbose)
 200         fprintf (stderr, "libnettle: using aes instructions.\n");
 201       nettle_aes128_encrypt_vec = _nettle_aes128_encrypt_aesni;
 202       nettle_aes128_decrypt_vec = _nettle_aes128_decrypt_aesni;
 203       nettle_aes192_encrypt_vec = _nettle_aes192_encrypt_aesni;
 204       nettle_aes192_decrypt_vec = _nettle_aes192_decrypt_aesni;
 205       nettle_aes256_encrypt_vec = _nettle_aes256_encrypt_aesni;
 206       nettle_aes256_decrypt_vec = _nettle_aes256_decrypt_aesni;
 207       nettle_cbc_aes128_encrypt_vec = _nettle_cbc_aes128_encrypt_aesni;
 208       nettle_cbc_aes192_encrypt_vec = _nettle_cbc_aes192_encrypt_aesni;
 209       nettle_cbc_aes256_encrypt_vec = _nettle_cbc_aes256_encrypt_aesni;
 210     }
 211   else
 212     {
 213       if (verbose)
 214         fprintf (stderr, "libnettle: not using aes instructions.\n");
 215       nettle_aes128_encrypt_vec = _nettle_aes128_encrypt_c;
 216       nettle_aes128_decrypt_vec = _nettle_aes128_decrypt_c;
 217       nettle_aes192_encrypt_vec = _nettle_aes192_encrypt_c;
 218       nettle_aes192_decrypt_vec = _nettle_aes192_decrypt_c;
 219       nettle_aes256_encrypt_vec = _nettle_aes256_encrypt_c;
 220       nettle_aes256_decrypt_vec = _nettle_aes256_decrypt_c;
 221       nettle_cbc_aes128_encrypt_vec = _nettle_cbc_aes128_encrypt_c;
 222       nettle_cbc_aes192_encrypt_vec = _nettle_cbc_aes192_encrypt_c;
 223       nettle_cbc_aes256_encrypt_vec = _nettle_cbc_aes256_encrypt_c;
 224     }
 225
 226   if (features.have_sha_ni)
 227     {
 228       if (verbose)
 229         fprintf (stderr, "libnettle: using sha_ni instructions.\n");
 230       nettle_sha1_compress_vec = _nettle_sha1_compress_sha_ni;
 231       _nettle_sha256_compress_n_vec = _nettle_sha256_compress_n_sha_ni;
 232     }
 233   else
 234     {
 235       if (verbose)
 236         fprintf (stderr, "libnettle: not using sha_ni instructions.\n");
 237       nettle_sha1_compress_vec = _nettle_sha1_compress_x86_64;
 238       _nettle_sha256_compress_n_vec = _nettle_sha256_compress_n_x86_64;
 239     }
 240
 241   if (features.have_pclmul)
 242     {
 243       if (verbose)
 244         fprintf (stderr, "libnettle: using pclmulqdq instructions.\n");
 245       _nettle_ghash_set_key_vec = _nettle_ghash_set_key_pclmul;
 246       _nettle_ghash_update_vec = _nettle_ghash_update_pclmul;
 247     }
 248   else
 249     {
 250       if (verbose)
 251         fprintf (stderr, "libnettle: not using pclmulqdq instructions.\n");
 252       _nettle_ghash_set_key_vec = _nettle_ghash_set_key_c;
 253       _nettle_ghash_update_vec = _nettle_ghash_update_table;
 254     }
 255
 256   if (features.vendor == X86_INTEL)
 257     {
 258       if (verbose)
 259         fprintf (stderr, "libnettle: intel SSE2 will be used for memxor.\n");
 260       nettle_memxor_vec = _nettle_memxor_sse2;
 261     }
 262   else
 263     {
 264       if (verbose)
 265         fprintf (stderr, "libnettle: intel SSE2 will not be used for memxor.\n");
 266       nettle_memxor_vec = _nettle_memxor_x86_64;
 267     }
 268 }
 269
 270 DEFINE_FAT_FUNC(nettle_aes128_encrypt, void,
 271  (const struct aes128_ctx *ctx, size_t length,
 272   uint8_t *dst,const uint8_t *src),
 273  (ctx, length, dst, src))
 274 DEFINE_FAT_FUNC(nettle_aes128_decrypt, void,
 275  (const struct aes128_ctx *ctx, size_t length,
 276   uint8_t *dst,const uint8_t *src),
 277  (ctx, length, dst, src))
 278
 279 DEFINE_FAT_FUNC(nettle_aes192_encrypt, void,
 280  (const struct aes192_ctx *ctx, size_t length,
 281   uint8_t *dst,const uint8_t *src),
 282  (ctx, length, dst, src))
 283 DEFINE_FAT_FUNC(nettle_aes192_decrypt, void,
 284  (const struct aes192_ctx *ctx, size_t length,
 285   uint8_t *dst,const uint8_t *src),
 286  (ctx, length, dst, src))
 287
 288 DEFINE_FAT_FUNC(nettle_aes256_encrypt, void,
 289  (const struct aes256_ctx *ctx, size_t length,
 290   uint8_t *dst,const uint8_t *src),
 291  (ctx, length, dst, src))
 292 DEFINE_FAT_FUNC(nettle_aes256_decrypt, void,
 293  (const struct aes256_ctx *ctx, size_t length,
 294   uint8_t *dst,const uint8_t *src),
 295  (ctx, length, dst, src))
 296
 297 DEFINE_FAT_FUNC(nettle_cbc_aes128_encrypt, void,
 298  (const struct aes128_ctx *ctx, uint8_t *iv,
 299   size_t length, uint8_t *dst, const uint8_t *src),
 300  (ctx, iv, length, dst, src))
 301 DEFINE_FAT_FUNC(nettle_cbc_aes192_encrypt, void,
 302  (const struct aes192_ctx *ctx, uint8_t *iv,
 303   size_t length, uint8_t *dst, const uint8_t *src),
 304  (ctx, iv, length, dst, src))
 305 DEFINE_FAT_FUNC(nettle_cbc_aes256_encrypt, void,
 306  (const struct aes256_ctx *ctx, uint8_t *iv,
 307   size_t length, uint8_t *dst, const uint8_t *src),
 308  (ctx, iv, length, dst, src))
 309
 310 DEFINE_FAT_FUNC(nettle_memxor, void *,
 311                 (void *dst, const void *src, size_t n),
 312                 (dst, src, n))
 313
 314 DEFINE_FAT_FUNC(nettle_sha1_compress, void,
 315                 (uint32_t *state, const uint8_t *input),
 316                 (state, input))
 317
 318 DEFINE_FAT_FUNC(_nettle_sha256_compress_n, const uint8_t *,
 319                 (uint32_t *state, const uint32_t *k,
 320                  size_t blocks, const uint8_t *input),
 321                 (state, k, blocks, input))
 322
 323 DEFINE_FAT_FUNC(_nettle_ghash_set_key, void,
 324                 (struct gcm_key *ctx, const union nettle_block16 *key),
 325                 (ctx, key))
 326 DEFINE_FAT_FUNC(_nettle_ghash_update, const uint8_t *,
 327                 (const struct gcm_key *ctx, union nettle_block16 *state,
 328                  size_t blocks, const uint8_t *data),
 329                 (ctx, state, blocks, data))