drivers/nvme/host/auth.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2020 Hannes Reinecke, SUSE Linux
   4  */
   5
   6 #include <linux/crc32.h>
   7 #include <linux/base64.h>
   8 #include <linux/prandom.h>
   9 #include <asm/unaligned.h>
  10 #include <crypto/hash.h>
  11 #include <crypto/dh.h>
  12 #include "nvme.h"
  13 #include "fabrics.h"
  14 #include <linux/nvme-auth.h>
  15
  16 #define CHAP_BUF_SIZE 4096
  17 static struct kmem_cache *nvme_chap_buf_cache;
  18 static mempool_t *nvme_chap_buf_pool;
  19
  20 struct nvme_dhchap_queue_context {
  21         struct list_head entry;
  22         struct work_struct auth_work;
  23         struct nvme_ctrl *ctrl;
  24         struct crypto_shash *shash_tfm;
  25         struct crypto_kpp *dh_tfm;
  26         struct nvme_dhchap_key *transformed_key;
  27         void *buf;
  28         int qid;
  29         int error;
  30         u32 s1;
  31         u32 s2;
  32         bool bi_directional;
  33         u16 transaction;
  34         u8 status;
  35         u8 dhgroup_id;
  36         u8 hash_id;
  37         size_t hash_len;
  38         u8 c1[64];
  39         u8 c2[64];
  40         u8 response[64];
  41         u8 *ctrl_key;
  42         u8 *host_key;
  43         u8 *sess_key;
  44         int ctrl_key_len;
  45         int host_key_len;
  46         int sess_key_len;
  47 };
  48
  49 static struct workqueue_struct *nvme_auth_wq;
  50
  51 #define nvme_auth_flags_from_qid(qid) \
  52         (qid == 0) ? 0 : BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED
  53 #define nvme_auth_queue_from_qid(ctrl, qid) \
  54         (qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q
  55
  56 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)
  57 {
  58         return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +
  59                         ctrl->opts->nr_poll_queues + 1;
  60 }
  61
  62 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
  63                             void *data, size_t data_len, bool auth_send)
  64 {
  65         struct nvme_command cmd = {};
  66         blk_mq_req_flags_t flags = nvme_auth_flags_from_qid(qid);
  67         struct request_queue *q = nvme_auth_queue_from_qid(ctrl, qid);
  68         int ret;
  69
  70         cmd.auth_common.opcode = nvme_fabrics_command;
  71         cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER;
  72         cmd.auth_common.spsp0 = 0x01;
  73         cmd.auth_common.spsp1 = 0x01;
  74         if (auth_send) {
  75                 cmd.auth_send.fctype = nvme_fabrics_type_auth_send;
  76                 cmd.auth_send.tl = cpu_to_le32(data_len);
  77         } else {
  78                 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive;
  79                 cmd.auth_receive.al = cpu_to_le32(data_len);
  80         }
  81
  82         ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
  83                                      qid == 0 ? NVME_QID_ANY : qid,
  84                                      0, flags);
  85         if (ret > 0)
  86                 dev_warn(ctrl->device,
  87                         "qid %d auth_send failed with status %d\n", qid, ret);
  88         else if (ret < 0)
  89                 dev_err(ctrl->device,
  90                         "qid %d auth_send failed with error %d\n", qid, ret);
  91         return ret;
  92 }
  93
  94 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,
  95                 struct nvmf_auth_dhchap_failure_data *data,
  96                 u16 transaction, u8 expected_msg)
  97 {
  98         dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",
  99                 __func__, qid, data->auth_type, data->auth_id);
 100
 101         if (data->auth_type == NVME_AUTH_COMMON_MESSAGES &&
 102             data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
 103                 return data->rescode_exp;
 104         }
 105         if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES ||
 106             data->auth_id != expected_msg) {
 107                 dev_warn(ctrl->device,
 108                          "qid %d invalid message %02x/%02x\n",
 109                          qid, data->auth_type, data->auth_id);
 110                 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 111         }
 112         if (le16_to_cpu(data->t_id) != transaction) {
 113                 dev_warn(ctrl->device,
 114                          "qid %d invalid transaction ID %d\n",
 115                          qid, le16_to_cpu(data->t_id));
 116                 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 117         }
 118         return 0;
 119 }
 120
 121 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,
 122                 struct nvme_dhchap_queue_context *chap)
 123 {
 124         struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;
 125         size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);
 126
 127         if (size > CHAP_BUF_SIZE) {
 128                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 129                 return -EINVAL;
 130         }
 131         memset((u8 *)chap->buf, 0, size);
 132         data->auth_type = NVME_AUTH_COMMON_MESSAGES;
 133         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
 134         data->t_id = cpu_to_le16(chap->transaction);
 135         data->sc_c = 0; /* No secure channel concatenation */
 136         data->napd = 1;
 137         data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID;
 138         data->auth_protocol[0].dhchap.halen = 3;
 139         data->auth_protocol[0].dhchap.dhlen = 6;
 140         data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256;
 141         data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384;
 142         data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512;
 143         data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL;
 144         data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048;
 145         data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072;
 146         data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096;
 147         data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144;
 148         data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192;
 149
 150         return size;
 151 }
 152
 153 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,
 154                 struct nvme_dhchap_queue_context *chap)
 155 {
 156         struct nvmf_auth_dhchap_challenge_data *data = chap->buf;
 157         u16 dhvlen = le16_to_cpu(data->dhvlen);
 158         size_t size = sizeof(*data) + data->hl + dhvlen;
 159         const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);
 160         const char *hmac_name, *kpp_name;
 161
 162         if (size > CHAP_BUF_SIZE) {
 163                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 164                 return -EINVAL;
 165         }
 166
 167         hmac_name = nvme_auth_hmac_name(data->hashid);
 168         if (!hmac_name) {
 169                 dev_warn(ctrl->device,
 170                          "qid %d: invalid HASH ID %d\n",
 171                          chap->qid, data->hashid);
 172                 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 173                 return -EPROTO;
 174         }
 175
 176         if (chap->hash_id == data->hashid && chap->shash_tfm &&
 177             !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) &&
 178             crypto_shash_digestsize(chap->shash_tfm) == data->hl) {
 179                 dev_dbg(ctrl->device,
 180                         "qid %d: reuse existing hash %s\n",
 181                         chap->qid, hmac_name);
 182                 goto select_kpp;
 183         }
 184
 185         /* Reset if hash cannot be reused */
 186         if (chap->shash_tfm) {
 187                 crypto_free_shash(chap->shash_tfm);
 188                 chap->hash_id = 0;
 189                 chap->hash_len = 0;
 190         }
 191         chap->shash_tfm = crypto_alloc_shash(hmac_name, 0,
 192                                              CRYPTO_ALG_ALLOCATES_MEMORY);
 193         if (IS_ERR(chap->shash_tfm)) {
 194                 dev_warn(ctrl->device,
 195                          "qid %d: failed to allocate hash %s, error %ld\n",
 196                          chap->qid, hmac_name, PTR_ERR(chap->shash_tfm));
 197                 chap->shash_tfm = NULL;
 198                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 199                 return -ENOMEM;
 200         }
 201
 202         if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) {
 203                 dev_warn(ctrl->device,
 204                          "qid %d: invalid hash length %d\n",
 205                          chap->qid, data->hl);
 206                 crypto_free_shash(chap->shash_tfm);
 207                 chap->shash_tfm = NULL;
 208                 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 209                 return -EPROTO;
 210         }
 211
 212         chap->hash_id = data->hashid;
 213         chap->hash_len = data->hl;
 214         dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
 215                 chap->qid, hmac_name);
 216
 217 select_kpp:
 218         kpp_name = nvme_auth_dhgroup_kpp(data->dhgid);
 219         if (!kpp_name) {
 220                 dev_warn(ctrl->device,
 221                          "qid %d: invalid DH group id %d\n",
 222                          chap->qid, data->dhgid);
 223                 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 224                 /* Leave previous dh_tfm intact */
 225                 return -EPROTO;
 226         }
 227
 228         if (chap->dhgroup_id == data->dhgid &&
 229             (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {
 230                 dev_dbg(ctrl->device,
 231                         "qid %d: reuse existing DH group %s\n",
 232                         chap->qid, gid_name);
 233                 goto skip_kpp;
 234         }
 235
 236         /* Reset dh_tfm if it can't be reused */
 237         if (chap->dh_tfm) {
 238                 crypto_free_kpp(chap->dh_tfm);
 239                 chap->dh_tfm = NULL;
 240         }
 241
 242         if (data->dhgid != NVME_AUTH_DHGROUP_NULL) {
 243                 if (dhvlen == 0) {
 244                         dev_warn(ctrl->device,
 245                                  "qid %d: empty DH value\n",
 246                                  chap->qid);
 247                         chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 248                         return -EPROTO;
 249                 }
 250
 251                 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0);
 252                 if (IS_ERR(chap->dh_tfm)) {
 253                         int ret = PTR_ERR(chap->dh_tfm);
 254
 255                         dev_warn(ctrl->device,
 256                                  "qid %d: error %d initializing DH group %s\n",
 257                                  chap->qid, ret, gid_name);
 258                         chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 259                         chap->dh_tfm = NULL;
 260                         return ret;
 261                 }
 262                 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",
 263                         chap->qid, gid_name);
 264         } else if (dhvlen != 0) {
 265                 dev_warn(ctrl->device,
 266                          "qid %d: invalid DH value for NULL DH\n",
 267                          chap->qid);
 268                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 269                 return -EPROTO;
 270         }
 271         chap->dhgroup_id = data->dhgid;
 272
 273 skip_kpp:
 274         chap->s1 = le32_to_cpu(data->seqnum);
 275         memcpy(chap->c1, data->cval, chap->hash_len);
 276         if (dhvlen) {
 277                 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL);
 278                 if (!chap->ctrl_key) {
 279                         chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 280                         return -ENOMEM;
 281                 }
 282                 chap->ctrl_key_len = dhvlen;
 283                 memcpy(chap->ctrl_key, data->cval + chap->hash_len,
 284                        dhvlen);
 285                 dev_dbg(ctrl->device, "ctrl public key %*ph\n",
 286                          (int)chap->ctrl_key_len, chap->ctrl_key);
 287         }
 288
 289         return 0;
 290 }
 291
 292 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,
 293                 struct nvme_dhchap_queue_context *chap)
 294 {
 295         struct nvmf_auth_dhchap_reply_data *data = chap->buf;
 296         size_t size = sizeof(*data);
 297
 298         size += 2 * chap->hash_len;
 299
 300         if (chap->host_key_len)
 301                 size += chap->host_key_len;
 302
 303         if (size > CHAP_BUF_SIZE) {
 304                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 305                 return -EINVAL;
 306         }
 307
 308         memset(chap->buf, 0, size);
 309         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 310         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
 311         data->t_id = cpu_to_le16(chap->transaction);
 312         data->hl = chap->hash_len;
 313         data->dhvlen = cpu_to_le16(chap->host_key_len);
 314         memcpy(data->rval, chap->response, chap->hash_len);
 315         if (ctrl->ctrl_key) {
 316                 chap->bi_directional = true;
 317                 get_random_bytes(chap->c2, chap->hash_len);
 318                 data->cvalid = 1;
 319                 memcpy(data->rval + chap->hash_len, chap->c2,
 320                        chap->hash_len);
 321                 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",
 322                         __func__, chap->qid, (int)chap->hash_len, chap->c2);
 323         } else {
 324                 memset(chap->c2, 0, chap->hash_len);
 325         }
 326         chap->s2 = nvme_auth_get_seqnum();
 327         data->seqnum = cpu_to_le32(chap->s2);
 328         if (chap->host_key_len) {
 329                 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",
 330                         __func__, chap->qid,
 331                         chap->host_key_len, chap->host_key);
 332                 memcpy(data->rval + 2 * chap->hash_len, chap->host_key,
 333                        chap->host_key_len);
 334         }
 335
 336         return size;
 337 }
 338
 339 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
 340                 struct nvme_dhchap_queue_context *chap)
 341 {
 342         struct nvmf_auth_dhchap_success1_data *data = chap->buf;
 343         size_t size = sizeof(*data) + chap->hash_len;
 344
 345         if (size > CHAP_BUF_SIZE) {
 346                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 347                 return -EINVAL;
 348         }
 349
 350         if (data->hl != chap->hash_len) {
 351                 dev_warn(ctrl->device,
 352                          "qid %d: invalid hash length %u\n",
 353                          chap->qid, data->hl);
 354                 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 355                 return -EPROTO;
 356         }
 357
 358         /* Just print out information for the admin queue */
 359         if (chap->qid == 0)
 360                 dev_info(ctrl->device,
 361                          "qid 0: authenticated with hash %s dhgroup %s\n",
 362                          nvme_auth_hmac_name(chap->hash_id),
 363                          nvme_auth_dhgroup_name(chap->dhgroup_id));
 364
 365         if (!data->rvalid)
 366                 return 0;
 367
 368         /* Validate controller response */
 369         if (memcmp(chap->response, data->rval, data->hl)) {
 370                 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",
 371                         __func__, chap->qid, (int)chap->hash_len, data->rval);
 372                 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",
 373                         __func__, chap->qid, (int)chap->hash_len,
 374                         chap->response);
 375                 dev_warn(ctrl->device,
 376                          "qid %d: controller authentication failed\n",
 377                          chap->qid);
 378                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 379                 return -ECONNREFUSED;
 380         }
 381
 382         /* Just print out information for the admin queue */
 383         if (chap->qid == 0)
 384                 dev_info(ctrl->device,
 385                          "qid 0: controller authenticated\n");
 386         return 0;
 387 }
 388
 389 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,
 390                 struct nvme_dhchap_queue_context *chap)
 391 {
 392         struct nvmf_auth_dhchap_success2_data *data = chap->buf;
 393         size_t size = sizeof(*data);
 394
 395         memset(chap->buf, 0, size);
 396         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 397         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
 398         data->t_id = cpu_to_le16(chap->transaction);
 399
 400         return size;
 401 }
 402
 403 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,
 404                 struct nvme_dhchap_queue_context *chap)
 405 {
 406         struct nvmf_auth_dhchap_failure_data *data = chap->buf;
 407         size_t size = sizeof(*data);
 408
 409         memset(chap->buf, 0, size);
 410         data->auth_type = NVME_AUTH_COMMON_MESSAGES;
 411         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
 412         data->t_id = cpu_to_le16(chap->transaction);
 413         data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
 414         data->rescode_exp = chap->status;
 415
 416         return size;
 417 }
 418
 419 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,
 420                 struct nvme_dhchap_queue_context *chap)
 421 {
 422         SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
 423         u8 buf[4], *challenge = chap->c1;
 424         int ret;
 425
 426         dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",
 427                 __func__, chap->qid, chap->s1, chap->transaction);
 428
 429         if (!chap->transformed_key) {
 430                 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,
 431                                                 ctrl->opts->host->nqn);
 432                 if (IS_ERR(chap->transformed_key)) {
 433                         ret = PTR_ERR(chap->transformed_key);
 434                         chap->transformed_key = NULL;
 435                         return ret;
 436                 }
 437         } else {
 438                 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",
 439                         __func__, chap->qid);
 440         }
 441
 442         ret = crypto_shash_setkey(chap->shash_tfm,
 443                         chap->transformed_key->key, chap->transformed_key->len);
 444         if (ret) {
 445                 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
 446                          chap->qid, ret);
 447                 goto out;
 448         }
 449
 450         if (chap->dh_tfm) {
 451                 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
 452                 if (!challenge) {
 453                         ret = -ENOMEM;
 454                         goto out;
 455                 }
 456                 ret = nvme_auth_augmented_challenge(chap->hash_id,
 457                                                     chap->sess_key,
 458                                                     chap->sess_key_len,
 459                                                     chap->c1, challenge,
 460                                                     chap->hash_len);
 461                 if (ret)
 462                         goto out;
 463         }
 464
 465         shash->tfm = chap->shash_tfm;
 466         ret = crypto_shash_init(shash);
 467         if (ret)
 468                 goto out;
 469         ret = crypto_shash_update(shash, challenge, chap->hash_len);
 470         if (ret)
 471                 goto out;
 472         put_unaligned_le32(chap->s1, buf);
 473         ret = crypto_shash_update(shash, buf, 4);
 474         if (ret)
 475                 goto out;
 476         put_unaligned_le16(chap->transaction, buf);
 477         ret = crypto_shash_update(shash, buf, 2);
 478         if (ret)
 479                 goto out;
 480         memset(buf, 0, sizeof(buf));
 481         ret = crypto_shash_update(shash, buf, 1);
 482         if (ret)
 483                 goto out;
 484         ret = crypto_shash_update(shash, "HostHost", 8);
 485         if (ret)
 486                 goto out;
 487         ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
 488                                   strlen(ctrl->opts->host->nqn));
 489         if (ret)
 490                 goto out;
 491         ret = crypto_shash_update(shash, buf, 1);
 492         if (ret)
 493                 goto out;
 494         ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
 495                             strlen(ctrl->opts->subsysnqn));
 496         if (ret)
 497                 goto out;
 498         ret = crypto_shash_final(shash, chap->response);
 499 out:
 500         if (challenge != chap->c1)
 501                 kfree(challenge);
 502         return ret;
 503 }
 504
 505 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
 506                 struct nvme_dhchap_queue_context *chap)
 507 {
 508         SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
 509         struct nvme_dhchap_key *transformed_key;
 510         u8 buf[4], *challenge = chap->c2;
 511         int ret;
 512
 513         transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,
 514                                 ctrl->opts->subsysnqn);
 515         if (IS_ERR(transformed_key)) {
 516                 ret = PTR_ERR(transformed_key);
 517                 return ret;
 518         }
 519
 520         ret = crypto_shash_setkey(chap->shash_tfm,
 521                         transformed_key->key, transformed_key->len);
 522         if (ret) {
 523                 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
 524                          chap->qid, ret);
 525                 goto out;
 526         }
 527
 528         if (chap->dh_tfm) {
 529                 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
 530                 if (!challenge) {
 531                         ret = -ENOMEM;
 532                         goto out;
 533                 }
 534                 ret = nvme_auth_augmented_challenge(chap->hash_id,
 535                                                     chap->sess_key,
 536                                                     chap->sess_key_len,
 537                                                     chap->c2, challenge,
 538                                                     chap->hash_len);
 539                 if (ret)
 540                         goto out;
 541         }
 542         dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",
 543                 __func__, chap->qid, chap->s2, chap->transaction);
 544         dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",
 545                 __func__, chap->qid, (int)chap->hash_len, challenge);
 546         dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",
 547                 __func__, chap->qid, ctrl->opts->subsysnqn);
 548         dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",
 549                 __func__, chap->qid, ctrl->opts->host->nqn);
 550         shash->tfm = chap->shash_tfm;
 551         ret = crypto_shash_init(shash);
 552         if (ret)
 553                 goto out;
 554         ret = crypto_shash_update(shash, challenge, chap->hash_len);
 555         if (ret)
 556                 goto out;
 557         put_unaligned_le32(chap->s2, buf);
 558         ret = crypto_shash_update(shash, buf, 4);
 559         if (ret)
 560                 goto out;
 561         put_unaligned_le16(chap->transaction, buf);
 562         ret = crypto_shash_update(shash, buf, 2);
 563         if (ret)
 564                 goto out;
 565         memset(buf, 0, 4);
 566         ret = crypto_shash_update(shash, buf, 1);
 567         if (ret)
 568                 goto out;
 569         ret = crypto_shash_update(shash, "Controller", 10);
 570         if (ret)
 571                 goto out;
 572         ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
 573                                   strlen(ctrl->opts->subsysnqn));
 574         if (ret)
 575                 goto out;
 576         ret = crypto_shash_update(shash, buf, 1);
 577         if (ret)
 578                 goto out;
 579         ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
 580                                   strlen(ctrl->opts->host->nqn));
 581         if (ret)
 582                 goto out;
 583         ret = crypto_shash_final(shash, chap->response);
 584 out:
 585         if (challenge != chap->c2)
 586                 kfree(challenge);
 587         nvme_auth_free_key(transformed_key);
 588         return ret;
 589 }
 590
 591 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
 592                 struct nvme_dhchap_queue_context *chap)
 593 {
 594         int ret;
 595
 596         if (chap->host_key && chap->host_key_len) {
 597                 dev_dbg(ctrl->device,
 598                         "qid %d: reusing host key\n", chap->qid);
 599                 goto gen_sesskey;
 600         }
 601         ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id);
 602         if (ret < 0) {
 603                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 604                 return ret;
 605         }
 606
 607         chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm);
 608
 609         chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL);
 610         if (!chap->host_key) {
 611                 chap->host_key_len = 0;
 612                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 613                 return -ENOMEM;
 614         }
 615         ret = nvme_auth_gen_pubkey(chap->dh_tfm,
 616                                    chap->host_key, chap->host_key_len);
 617         if (ret) {
 618                 dev_dbg(ctrl->device,
 619                         "failed to generate public key, error %d\n", ret);
 620                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 621                 return ret;
 622         }
 623
 624 gen_sesskey:
 625         chap->sess_key_len = chap->host_key_len;
 626         chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
 627         if (!chap->sess_key) {
 628                 chap->sess_key_len = 0;
 629                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 630                 return -ENOMEM;
 631         }
 632
 633         ret = nvme_auth_gen_shared_secret(chap->dh_tfm,
 634                                           chap->ctrl_key, chap->ctrl_key_len,
 635                                           chap->sess_key, chap->sess_key_len);
 636         if (ret) {
 637                 dev_dbg(ctrl->device,
 638                         "failed to generate shared secret, error %d\n", ret);
 639                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 640                 return ret;
 641         }
 642         dev_dbg(ctrl->device, "shared secret %*ph\n",
 643                 (int)chap->sess_key_len, chap->sess_key);
 644         return 0;
 645 }
 646
 647 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap)
 648 {
 649         nvme_auth_free_key(chap->transformed_key);
 650         chap->transformed_key = NULL;
 651         kfree_sensitive(chap->host_key);
 652         chap->host_key = NULL;
 653         chap->host_key_len = 0;
 654         kfree_sensitive(chap->ctrl_key);
 655         chap->ctrl_key = NULL;
 656         chap->ctrl_key_len = 0;
 657         kfree_sensitive(chap->sess_key);
 658         chap->sess_key = NULL;
 659         chap->sess_key_len = 0;
 660         chap->status = 0;
 661         chap->error = 0;
 662         chap->s1 = 0;
 663         chap->s2 = 0;
 664         chap->bi_directional = false;
 665         chap->transaction = 0;
 666         memset(chap->c1, 0, sizeof(chap->c1));
 667         memset(chap->c2, 0, sizeof(chap->c2));
 668         mempool_free(chap->buf, nvme_chap_buf_pool);
 669         chap->buf = NULL;
 670 }
 671
 672 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap)
 673 {
 674         nvme_auth_reset_dhchap(chap);
 675         if (chap->shash_tfm)
 676                 crypto_free_shash(chap->shash_tfm);
 677         if (chap->dh_tfm)
 678                 crypto_free_kpp(chap->dh_tfm);
 679 }
 680
 681 static void nvme_queue_auth_work(struct work_struct *work)
 682 {
 683         struct nvme_dhchap_queue_context *chap =
 684                 container_of(work, struct nvme_dhchap_queue_context, auth_work);
 685         struct nvme_ctrl *ctrl = chap->ctrl;
 686         size_t tl;
 687         int ret = 0;
 688
 689         /*
 690          * Allocate a large enough buffer for the entire negotiation:
 691          * 4k is enough to ffdhe8192.
 692          */
 693         chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL);
 694         if (!chap->buf) {
 695                 chap->error = -ENOMEM;
 696                 return;
 697         }
 698
 699         chap->transaction = ctrl->transaction++;
 700
 701         /* DH-HMAC-CHAP Step 1: send negotiate */
 702         dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",
 703                 __func__, chap->qid);
 704         ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);
 705         if (ret < 0) {
 706                 chap->error = ret;
 707                 return;
 708         }
 709         tl = ret;
 710         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 711         if (ret) {
 712                 chap->error = ret;
 713                 return;
 714         }
 715
 716         /* DH-HMAC-CHAP Step 2: receive challenge */
 717         dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",
 718                 __func__, chap->qid);
 719
 720         memset(chap->buf, 0, CHAP_BUF_SIZE);
 721         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
 722                                false);
 723         if (ret) {
 724                 dev_warn(ctrl->device,
 725                          "qid %d failed to receive challenge, %s %d\n",
 726                          chap->qid, ret < 0 ? "error" : "nvme status", ret);
 727                 chap->error = ret;
 728                 return;
 729         }
 730         ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,
 731                                          NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE);
 732         if (ret) {
 733                 chap->status = ret;
 734                 chap->error = -ECONNREFUSED;
 735                 return;
 736         }
 737
 738         ret = nvme_auth_process_dhchap_challenge(ctrl, chap);
 739         if (ret) {
 740                 /* Invalid challenge parameters */
 741                 chap->error = ret;
 742                 goto fail2;
 743         }
 744
 745         if (chap->ctrl_key_len) {
 746                 dev_dbg(ctrl->device,
 747                         "%s: qid %d DH exponential\n",
 748                         __func__, chap->qid);
 749                 ret = nvme_auth_dhchap_exponential(ctrl, chap);
 750                 if (ret) {
 751                         chap->error = ret;
 752                         goto fail2;
 753                 }
 754         }
 755
 756         dev_dbg(ctrl->device, "%s: qid %d host response\n",
 757                 __func__, chap->qid);
 758         mutex_lock(&ctrl->dhchap_auth_mutex);
 759         ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);
 760         if (ret) {
 761                 mutex_unlock(&ctrl->dhchap_auth_mutex);
 762                 chap->error = ret;
 763                 goto fail2;
 764         }
 765         mutex_unlock(&ctrl->dhchap_auth_mutex);
 766
 767         /* DH-HMAC-CHAP Step 3: send reply */
 768         dev_dbg(ctrl->device, "%s: qid %d send reply\n",
 769                 __func__, chap->qid);
 770         ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);
 771         if (ret < 0) {
 772                 chap->error = ret;
 773                 goto fail2;
 774         }
 775
 776         tl = ret;
 777         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 778         if (ret) {
 779                 chap->error = ret;
 780                 goto fail2;
 781         }
 782
 783         /* DH-HMAC-CHAP Step 4: receive success1 */
 784         dev_dbg(ctrl->device, "%s: qid %d receive success1\n",
 785                 __func__, chap->qid);
 786
 787         memset(chap->buf, 0, CHAP_BUF_SIZE);
 788         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
 789                                false);
 790         if (ret) {
 791                 dev_warn(ctrl->device,
 792                          "qid %d failed to receive success1, %s %d\n",
 793                          chap->qid, ret < 0 ? "error" : "nvme status", ret);
 794                 chap->error = ret;
 795                 return;
 796         }
 797         ret = nvme_auth_receive_validate(ctrl, chap->qid,
 798                                          chap->buf, chap->transaction,
 799                                          NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1);
 800         if (ret) {
 801                 chap->status = ret;
 802                 chap->error = -ECONNREFUSED;
 803                 return;
 804         }
 805
 806         mutex_lock(&ctrl->dhchap_auth_mutex);
 807         if (ctrl->ctrl_key) {
 808                 dev_dbg(ctrl->device,
 809                         "%s: qid %d controller response\n",
 810                         __func__, chap->qid);
 811                 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);
 812                 if (ret) {
 813                         mutex_unlock(&ctrl->dhchap_auth_mutex);
 814                         chap->error = ret;
 815                         goto fail2;
 816                 }
 817         }
 818         mutex_unlock(&ctrl->dhchap_auth_mutex);
 819
 820         ret = nvme_auth_process_dhchap_success1(ctrl, chap);
 821         if (ret) {
 822                 /* Controller authentication failed */
 823                 chap->error = -ECONNREFUSED;
 824                 goto fail2;
 825         }
 826
 827         if (chap->bi_directional) {
 828                 /* DH-HMAC-CHAP Step 5: send success2 */
 829                 dev_dbg(ctrl->device, "%s: qid %d send success2\n",
 830                         __func__, chap->qid);
 831                 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);
 832                 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 833                 if (ret)
 834                         chap->error = ret;
 835         }
 836         if (!ret) {
 837                 chap->error = 0;
 838                 return;
 839         }
 840
 841 fail2:
 842         dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",
 843                 __func__, chap->qid, chap->status);
 844         tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);
 845         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 846         /*
 847          * only update error if send failure2 failed and no other
 848          * error had been set during authentication.
 849          */
 850         if (ret && !chap->error)
 851                 chap->error = ret;
 852 }
 853
 854 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
 855 {
 856         struct nvme_dhchap_queue_context *chap;
 857
 858         if (!ctrl->host_key) {
 859                 dev_warn(ctrl->device, "qid %d: no key\n", qid);
 860                 return -ENOKEY;
 861         }
 862
 863         if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {
 864                 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);
 865                 return -ENOKEY;
 866         }
 867
 868         chap = &ctrl->dhchap_ctxs[qid];
 869         cancel_work_sync(&chap->auth_work);
 870         queue_work(nvme_auth_wq, &chap->auth_work);
 871         return 0;
 872 }
 873 EXPORT_SYMBOL_GPL(nvme_auth_negotiate);
 874
 875 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
 876 {
 877         struct nvme_dhchap_queue_context *chap;
 878         int ret;
 879
 880         chap = &ctrl->dhchap_ctxs[qid];
 881         flush_work(&chap->auth_work);
 882         ret = chap->error;
 883         /* clear sensitive info */
 884         nvme_auth_reset_dhchap(chap);
 885         return ret;
 886 }
 887 EXPORT_SYMBOL_GPL(nvme_auth_wait);
 888
 889 static void nvme_ctrl_auth_work(struct work_struct *work)
 890 {
 891         struct nvme_ctrl *ctrl =
 892                 container_of(work, struct nvme_ctrl, dhchap_auth_work);
 893         int ret, q;
 894
 895         /*
 896          * If the ctrl is no connected, bail as reconnect will handle
 897          * authentication.
 898          */
 899         if (ctrl->state != NVME_CTRL_LIVE)
 900                 return;
 901
 902         /* Authenticate admin queue first */
 903         ret = nvme_auth_negotiate(ctrl, 0);
 904         if (ret) {
 905                 dev_warn(ctrl->device,
 906                          "qid 0: error %d setting up authentication\n", ret);
 907                 return;
 908         }
 909         ret = nvme_auth_wait(ctrl, 0);
 910         if (ret) {
 911                 dev_warn(ctrl->device,
 912                          "qid 0: authentication failed\n");
 913                 return;
 914         }
 915
 916         for (q = 1; q < ctrl->queue_count; q++) {
 917                 ret = nvme_auth_negotiate(ctrl, q);
 918                 if (ret) {
 919                         dev_warn(ctrl->device,
 920                                  "qid %d: error %d setting up authentication\n",
 921                                  q, ret);
 922                         break;
 923                 }
 924         }
 925
 926         /*
 927          * Failure is a soft-state; credentials remain valid until
 928          * the controller terminates the connection.
 929          */
 930         for (q = 1; q < ctrl->queue_count; q++) {
 931                 ret = nvme_auth_wait(ctrl, q);
 932                 if (ret)
 933                         dev_warn(ctrl->device,
 934                                  "qid %d: authentication failed\n", q);
 935         }
 936 }
 937
 938 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)
 939 {
 940         struct nvme_dhchap_queue_context *chap;
 941         int i, ret;
 942
 943         mutex_init(&ctrl->dhchap_auth_mutex);
 944         INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);
 945         if (!ctrl->opts)
 946                 return 0;
 947         ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,
 948                         &ctrl->host_key);
 949         if (ret)
 950                 return ret;
 951         ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,
 952                         &ctrl->ctrl_key);
 953         if (ret)
 954                 goto err_free_dhchap_secret;
 955
 956         if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)
 957                 return 0;
 958
 959         ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),
 960                                 sizeof(*chap), GFP_KERNEL);
 961         if (!ctrl->dhchap_ctxs) {
 962                 ret = -ENOMEM;
 963                 goto err_free_dhchap_ctrl_secret;
 964         }
 965
 966         for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {
 967                 chap = &ctrl->dhchap_ctxs[i];
 968                 chap->qid = i;
 969                 chap->ctrl = ctrl;
 970                 INIT_WORK(&chap->auth_work, nvme_queue_auth_work);
 971         }
 972
 973         return 0;
 974 err_free_dhchap_ctrl_secret:
 975         nvme_auth_free_key(ctrl->ctrl_key);
 976         ctrl->ctrl_key = NULL;
 977 err_free_dhchap_secret:
 978         nvme_auth_free_key(ctrl->host_key);
 979         ctrl->host_key = NULL;
 980         return ret;
 981 }
 982 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);
 983
 984 void nvme_auth_stop(struct nvme_ctrl *ctrl)
 985 {
 986         cancel_work_sync(&ctrl->dhchap_auth_work);
 987 }
 988 EXPORT_SYMBOL_GPL(nvme_auth_stop);
 989
 990 void nvme_auth_free(struct nvme_ctrl *ctrl)
 991 {
 992         int i;
 993
 994         if (ctrl->dhchap_ctxs) {
 995                 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)
 996                         nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);
 997                 kfree(ctrl->dhchap_ctxs);
 998         }
 999         if (ctrl->host_key) {
1000                 nvme_auth_free_key(ctrl->host_key);
1001                 ctrl->host_key = NULL;
1002         }
1003         if (ctrl->ctrl_key) {
1004                 nvme_auth_free_key(ctrl->ctrl_key);
1005                 ctrl->ctrl_key = NULL;
1006         }
1007 }
1008 EXPORT_SYMBOL_GPL(nvme_auth_free);
1009
1010 int __init nvme_init_auth(void)
1011 {
1012         nvme_auth_wq = alloc_workqueue("nvme-auth-wq",
1013                                WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
1014         if (!nvme_auth_wq)
1015                 return -ENOMEM;
1016
1017         nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache",
1018                                 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL);
1019         if (!nvme_chap_buf_cache)
1020                 goto err_destroy_workqueue;
1021
1022         nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab,
1023                         mempool_free_slab, nvme_chap_buf_cache);
1024         if (!nvme_chap_buf_pool)
1025                 goto err_destroy_chap_buf_cache;
1026
1027         return 0;
1028 err_destroy_chap_buf_cache:
1029         kmem_cache_destroy(nvme_chap_buf_cache);
1030 err_destroy_workqueue:
1031         destroy_workqueue(nvme_auth_wq);
1032         return -ENOMEM;
1033 }
1034
1035 void __exit nvme_exit_auth(void)
1036 {
1037         mempool_destroy(nvme_chap_buf_pool);
1038         kmem_cache_destroy(nvme_chap_buf_cache);
1039         destroy_workqueue(nvme_auth_wq);
1040 }