git.ipfire.org Git - thirdparty/kernel/stable.git/blob

1 // SPDX-License-Identifier: GPL-2.0

4 #include <linux/kernel.h>

5 #include <linux/module.h>

6 #include <crypto/algapi.h>

7 #include <crypto/internal/aead.h>

8 #include <crypto/authenc.h>

9 #include <crypto/internal/des.h>

10 #include <linux/rtnetlink.h>

11 #include "cc_driver.h"

12 #include "cc_buffer_mgr.h"

13 #include "cc_aead.h"

14 #include "cc_request_mgr.h"

15 #include "cc_hash.h"

16 #include "cc_sram_mgr.h"

18 #define template_aead template_u.aead

20 #define MAX_AEAD_SETKEY_SEQ 12

21 #define MAX_AEAD_PROCESS_SEQ 23

23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)

24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)

26 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE

28 struct cc_aead_handle {

29 cc_sram_addr_t sram_workspace_addr;

30 struct list_head aead_list;

31 };

33 struct cc_hmac_s {

34 u8 *padded_authkey;

35 u8 *ipad_opad; /* IPAD, OPAD*/

36 dma_addr_t padded_authkey_dma_addr;

37 dma_addr_t ipad_opad_dma_addr;

38 };

40 struct cc_xcbc_s {

41 u8 *xcbc_keys; /* K1,K2,K3 */

42 dma_addr_t xcbc_keys_dma_addr;

43 };

45 struct cc_aead_ctx {

46 struct cc_drvdata *drvdata;

47 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */

48 u8 *enckey;

49 dma_addr_t enckey_dma_addr;

50 union {

51 struct cc_hmac_s hmac;

52 struct cc_xcbc_s xcbc;

53 } auth_state;

54 unsigned int enc_keylen;

55 unsigned int auth_keylen;

56 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */

57 unsigned int hash_len;

58 enum drv_cipher_mode cipher_mode;

59 enum cc_flow_mode flow_mode;

60 enum drv_hash_mode auth_mode;

61 };

63 static inline bool valid_assoclen(struct aead_request *req)

64 {

65 return ((req->assoclen == 16) || (req->assoclen == 20));

66 }

68 static void cc_aead_exit(struct crypto_aead *tfm)

69 {

70 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

71 struct device *dev = drvdata_to_dev(ctx->drvdata);

73 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),

74 crypto_tfm_alg_name(&tfm->base));

76 /* Unmap enckey buffer */

77 if (ctx->enckey) {

78 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,

79 ctx->enckey_dma_addr);

80 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",

81 &ctx->enckey_dma_addr);

82 ctx->enckey_dma_addr = 0;

83 ctx->enckey = NULL;

84 }

86 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */

87 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;

89 if (xcbc->xcbc_keys) {

90 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,

91 xcbc->xcbc_keys,

92 xcbc->xcbc_keys_dma_addr);

93 }

94 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",

95 &xcbc->xcbc_keys_dma_addr);

96 xcbc->xcbc_keys_dma_addr = 0;

97 xcbc->xcbc_keys = NULL;

98 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */

99 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;

100

101 if (hmac->ipad_opad) {

102 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,

103 hmac->ipad_opad,

104 hmac->ipad_opad_dma_addr);

105 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",

106 &hmac->ipad_opad_dma_addr);

107 hmac->ipad_opad_dma_addr = 0;

108 hmac->ipad_opad = NULL;

109 }

110 if (hmac->padded_authkey) {

111 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,

112 hmac->padded_authkey,

113 hmac->padded_authkey_dma_addr);

114 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",

115 &hmac->padded_authkey_dma_addr);

116 hmac->padded_authkey_dma_addr = 0;

117 hmac->padded_authkey = NULL;

118 }

119 }

120 }

121

122 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm)

123 {

124 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

125

126 return cc_get_default_hash_len(ctx->drvdata);

127 }

128

129 static int cc_aead_init(struct crypto_aead *tfm)

130 {

131 struct aead_alg *alg = crypto_aead_alg(tfm);

132 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

133 struct cc_crypto_alg *cc_alg =

134 container_of(alg, struct cc_crypto_alg, aead_alg);

135 struct device *dev = drvdata_to_dev(cc_alg->drvdata);

136

137 dev_dbg(dev, "Initializing context @%p for %s\n", ctx,

138 crypto_tfm_alg_name(&tfm->base));

139

140 /* Initialize modes in instance */

141 ctx->cipher_mode = cc_alg->cipher_mode;

142 ctx->flow_mode = cc_alg->flow_mode;

143 ctx->auth_mode = cc_alg->auth_mode;

144 ctx->drvdata = cc_alg->drvdata;

145 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));

146

147 /* Allocate key buffer, cache line aligned */

148 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,

149 &ctx->enckey_dma_addr, GFP_KERNEL);

150 if (!ctx->enckey) {

151 dev_err(dev, "Failed allocating key buffer\n");

152 goto init_failed;

153 }

154 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",

155 ctx->enckey);

156

157 /* Set default authlen value */

158

159 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */

160 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;

161 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;

162

163 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */

164 /* (and temporary for user key - up to 256b) */

165 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,

166 &xcbc->xcbc_keys_dma_addr,

167 GFP_KERNEL);

168 if (!xcbc->xcbc_keys) {

169 dev_err(dev, "Failed allocating buffer for XCBC keys\n");

170 goto init_failed;

171 }

172 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */

173 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;

174 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;

175 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;

176

177 /* Allocate dma-coherent buffer for IPAD + OPAD */

178 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,

179 &hmac->ipad_opad_dma_addr,

180 GFP_KERNEL);

181

182 if (!hmac->ipad_opad) {

183 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");

184 goto init_failed;

185 }

186

187 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",

188 hmac->ipad_opad);

189

190 hmac->padded_authkey = dma_alloc_coherent(dev,

191 MAX_HMAC_BLOCK_SIZE,

192 pkey_dma,

193 GFP_KERNEL);

194

195 if (!hmac->padded_authkey) {

196 dev_err(dev, "failed to allocate padded_authkey\n");

197 goto init_failed;

198 }

199 } else {

200 ctx->auth_state.hmac.ipad_opad = NULL;

201 ctx->auth_state.hmac.padded_authkey = NULL;

202 }

203 ctx->hash_len = cc_get_aead_hash_len(tfm);

204

205 return 0;

206

207 init_failed:

208 cc_aead_exit(tfm);

209 return -ENOMEM;

210 }

211

212 static void cc_aead_complete(struct device *dev, void *cc_req, int err)

213 {

214 struct aead_request *areq = (struct aead_request *)cc_req;

215 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);

216 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);

217 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

218

219 /* BACKLOG notification */

220 if (err == -EINPROGRESS)

221 goto done;

222

223 cc_unmap_aead_request(dev, areq);

224

225 /* Restore ordinary iv pointer */

226 areq->iv = areq_ctx->backup_iv;

227

228 if (err)

229 goto done;

230

231 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {

232 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,

233 ctx->authsize) != 0) {

234 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",

235 ctx->authsize, ctx->cipher_mode);

236 /* In case of payload authentication failure, MUST NOT

237 * revealed the decrypted message --> zero its memory.

238 */

239 sg_zero_buffer(areq->dst, sg_nents(areq->dst),

240 areq->cryptlen, areq->assoclen);

241 err = -EBADMSG;

242 }

243 /*ENCRYPT*/

244 } else if (areq_ctx->is_icv_fragmented) {

245 u32 skip = areq->cryptlen + areq_ctx->dst_offset;

246

247 cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl,

248 skip, (skip + ctx->authsize),

249 CC_SG_FROM_BUF);

250 }

251 done:

252 aead_request_complete(areq, err);

253 }

254

255 static unsigned int xcbc_setkey(struct cc_hw_desc *desc,

256 struct cc_aead_ctx *ctx)

257 {

258 /* Load the AES key */

259 hw_desc_init(&desc[0]);

260 /* We are using for the source/user key the same buffer

261 * as for the output keys, * because after this key loading it

262 * is not needed anymore

263 */

264 set_din_type(&desc[0], DMA_DLLI,

265 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,

266 NS_BIT);

267 set_cipher_mode(&desc[0], DRV_CIPHER_ECB);

268 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);

269 set_key_size_aes(&desc[0], ctx->auth_keylen);

270 set_flow_mode(&desc[0], S_DIN_to_AES);

271 set_setup_mode(&desc[0], SETUP_LOAD_KEY0);

272

273 hw_desc_init(&desc[1]);

274 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);

275 set_flow_mode(&desc[1], DIN_AES_DOUT);

276 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,

277 AES_KEYSIZE_128, NS_BIT, 0);

278

279 hw_desc_init(&desc[2]);

280 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);

281 set_flow_mode(&desc[2], DIN_AES_DOUT);

282 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr

283 + AES_KEYSIZE_128),

284 AES_KEYSIZE_128, NS_BIT, 0);

285

286 hw_desc_init(&desc[3]);

287 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);

288 set_flow_mode(&desc[3], DIN_AES_DOUT);

289 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr

290 + 2 * AES_KEYSIZE_128),

291 AES_KEYSIZE_128, NS_BIT, 0);

292

293 return 4;

294 }

295

296 static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)

297 {

298 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };

299 unsigned int digest_ofs = 0;

300 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?

301 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;

302 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?

303 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;

304 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;

305

306 unsigned int idx = 0;

307 int i;

308

309 /* calc derived HMAC key */

310 for (i = 0; i < 2; i++) {

311 /* Load hash initial state */

312 hw_desc_init(&desc[idx]);

313 set_cipher_mode(&desc[idx], hash_mode);

314 set_din_sram(&desc[idx],

315 cc_larval_digest_addr(ctx->drvdata,

316 ctx->auth_mode),

317 digest_size);

318 set_flow_mode(&desc[idx], S_DIN_to_HASH);

319 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

320 idx++;

321

322 /* Load the hash current length*/

323 hw_desc_init(&desc[idx]);

324 set_cipher_mode(&desc[idx], hash_mode);

325 set_din_const(&desc[idx], 0, ctx->hash_len);

326 set_flow_mode(&desc[idx], S_DIN_to_HASH);

327 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

328 idx++;

329

330 /* Prepare ipad key */

331 hw_desc_init(&desc[idx]);

332 set_xor_val(&desc[idx], hmac_pad_const[i]);

333 set_cipher_mode(&desc[idx], hash_mode);

334 set_flow_mode(&desc[idx], S_DIN_to_HASH);

335 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

336 idx++;

337

338 /* Perform HASH update */

339 hw_desc_init(&desc[idx]);

340 set_din_type(&desc[idx], DMA_DLLI,

341 hmac->padded_authkey_dma_addr,

342 SHA256_BLOCK_SIZE, NS_BIT);

343 set_cipher_mode(&desc[idx], hash_mode);

344 set_xor_active(&desc[idx]);

345 set_flow_mode(&desc[idx], DIN_HASH);

346 idx++;

347

348 /* Get the digset */

349 hw_desc_init(&desc[idx]);

350 set_cipher_mode(&desc[idx], hash_mode);

351 set_dout_dlli(&desc[idx],

352 (hmac->ipad_opad_dma_addr + digest_ofs),

353 digest_size, NS_BIT, 0);

354 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

355 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

356 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);

357 idx++;

358

359 digest_ofs += digest_size;

360 }

361

362 return idx;

363 }

364

365 static int validate_keys_sizes(struct cc_aead_ctx *ctx)

366 {

367 struct device *dev = drvdata_to_dev(ctx->drvdata);

368

369 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n",

370 ctx->enc_keylen, ctx->auth_keylen);

371

372 switch (ctx->auth_mode) {

373 case DRV_HASH_SHA1:

374 case DRV_HASH_SHA256:

375 break;

376 case DRV_HASH_XCBC_MAC:

377 if (ctx->auth_keylen != AES_KEYSIZE_128 &&

378 ctx->auth_keylen != AES_KEYSIZE_192 &&

379 ctx->auth_keylen != AES_KEYSIZE_256)

380 return -ENOTSUPP;

381 break;

382 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */

383 if (ctx->auth_keylen > 0)

384 return -EINVAL;

385 break;

386 default:

387 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);

388 return -EINVAL;

389 }

390 /* Check cipher key size */

391 if (ctx->flow_mode == S_DIN_to_DES) {

392 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {

393 dev_err(dev, "Invalid cipher(3DES) key size: %u\n",

394 ctx->enc_keylen);

395 return -EINVAL;

396 }

397 } else { /* Default assumed to be AES ciphers */

398 if (ctx->enc_keylen != AES_KEYSIZE_128 &&

399 ctx->enc_keylen != AES_KEYSIZE_192 &&

400 ctx->enc_keylen != AES_KEYSIZE_256) {

401 dev_err(dev, "Invalid cipher(AES) key size: %u\n",

402 ctx->enc_keylen);

403 return -EINVAL;

404 }

405 }

406

407 return 0; /* All tests of keys sizes passed */

408 }

409

410 /* This function prepers the user key so it can pass to the hmac processing

411 * (copy to intenral buffer or hash in case of key longer than block

412 */

413 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey,

414 unsigned int keylen)

415 {

416 dma_addr_t key_dma_addr = 0;

417 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

418 struct device *dev = drvdata_to_dev(ctx->drvdata);

419 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);

420 struct cc_crypto_req cc_req = {};

421 unsigned int blocksize;

422 unsigned int digestsize;

423 unsigned int hashmode;

424 unsigned int idx = 0;

425 int rc = 0;

426 u8 *key = NULL;

427 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];

428 dma_addr_t padded_authkey_dma_addr =

429 ctx->auth_state.hmac.padded_authkey_dma_addr;

430

431 switch (ctx->auth_mode) { /* auth_key required and >0 */

432 case DRV_HASH_SHA1:

433 blocksize = SHA1_BLOCK_SIZE;

434 digestsize = SHA1_DIGEST_SIZE;

435 hashmode = DRV_HASH_HW_SHA1;

436 break;

437 case DRV_HASH_SHA256:

438 default:

439 blocksize = SHA256_BLOCK_SIZE;

440 digestsize = SHA256_DIGEST_SIZE;

441 hashmode = DRV_HASH_HW_SHA256;

442 }

443

444 if (keylen != 0) {

445

446 key = kmemdup(authkey, keylen, GFP_KERNEL);

447 if (!key)

448 return -ENOMEM;

449

450 key_dma_addr = dma_map_single(dev, (void *)key, keylen,

451 DMA_TO_DEVICE);

452 if (dma_mapping_error(dev, key_dma_addr)) {

453 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",

454 key, keylen);

455 kzfree(key);

456 return -ENOMEM;

457 }

458 if (keylen > blocksize) {

459 /* Load hash initial state */

460 hw_desc_init(&desc[idx]);

461 set_cipher_mode(&desc[idx], hashmode);

462 set_din_sram(&desc[idx], larval_addr, digestsize);

463 set_flow_mode(&desc[idx], S_DIN_to_HASH);

464 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

465 idx++;

466

467 /* Load the hash current length*/

468 hw_desc_init(&desc[idx]);

469 set_cipher_mode(&desc[idx], hashmode);

470 set_din_const(&desc[idx], 0, ctx->hash_len);

471 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);

472 set_flow_mode(&desc[idx], S_DIN_to_HASH);

473 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

474 idx++;

475

476 hw_desc_init(&desc[idx]);

477 set_din_type(&desc[idx], DMA_DLLI,

478 key_dma_addr, keylen, NS_BIT);

479 set_flow_mode(&desc[idx], DIN_HASH);

480 idx++;

481

482 /* Get hashed key */

483 hw_desc_init(&desc[idx]);

484 set_cipher_mode(&desc[idx], hashmode);

485 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,

486 digestsize, NS_BIT, 0);

487 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

488 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

489 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);

490 set_cipher_config0(&desc[idx],

491 HASH_DIGEST_RESULT_LITTLE_ENDIAN);

492 idx++;

493

494 hw_desc_init(&desc[idx]);

495 set_din_const(&desc[idx], 0, (blocksize - digestsize));

496 set_flow_mode(&desc[idx], BYPASS);

497 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +

498 digestsize), (blocksize - digestsize),

499 NS_BIT, 0);

500 idx++;

501 } else {

502 hw_desc_init(&desc[idx]);

503 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,

504 keylen, NS_BIT);

505 set_flow_mode(&desc[idx], BYPASS);

506 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,

507 keylen, NS_BIT, 0);

508 idx++;

509

510 if ((blocksize - keylen) != 0) {

511 hw_desc_init(&desc[idx]);

512 set_din_const(&desc[idx], 0,

513 (blocksize - keylen));

514 set_flow_mode(&desc[idx], BYPASS);

515 set_dout_dlli(&desc[idx],

516 (padded_authkey_dma_addr +

517 keylen),

518 (blocksize - keylen), NS_BIT, 0);

519 idx++;

520 }

521 }

522 } else {

523 hw_desc_init(&desc[idx]);

524 set_din_const(&desc[idx], 0, (blocksize - keylen));

525 set_flow_mode(&desc[idx], BYPASS);

526 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,

527 blocksize, NS_BIT, 0);

528 idx++;

529 }

530

531 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);

532 if (rc)

533 dev_err(dev, "send_request() failed (rc=%d)\n", rc);

534

535 if (key_dma_addr)

536 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);

537

538 kzfree(key);

539

540 return rc;

541 }

542

543 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,

544 unsigned int keylen)

545 {

546 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

547 struct cc_crypto_req cc_req = {};

548 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];

549 unsigned int seq_len = 0;

550 struct device *dev = drvdata_to_dev(ctx->drvdata);

551 const u8 *enckey, *authkey;

552 int rc;

553

554 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",

555 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);

556

557 /* STAT_PHASE_0: Init and sanity checks */

558

559 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */

560 struct crypto_authenc_keys keys;

561

562 rc = crypto_authenc_extractkeys(&keys, key, keylen);

563 if (rc)

564 goto badkey;

565 enckey = keys.enckey;

566 authkey = keys.authkey;

567 ctx->enc_keylen = keys.enckeylen;

568 ctx->auth_keylen = keys.authkeylen;

569

570 if (ctx->cipher_mode == DRV_CIPHER_CTR) {

571 /* the nonce is stored in bytes at end of key */

572 rc = -EINVAL;

573 if (ctx->enc_keylen <

574 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))

575 goto badkey;

576 /* Copy nonce from last 4 bytes in CTR key to

577 * first 4 bytes in CTR IV

578 */

579 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -

580 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);

581 /* Set CTR key size */

582 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;

583 }

584 } else { /* non-authenc - has just one key */

585 enckey = key;

586 authkey = NULL;

587 ctx->enc_keylen = keylen;

588 ctx->auth_keylen = 0;

589 }

590

591 rc = validate_keys_sizes(ctx);

592 if (rc)

593 goto badkey;

594

595 /* STAT_PHASE_1: Copy key to ctx */

596

597 /* Get key material */

598 memcpy(ctx->enckey, enckey, ctx->enc_keylen);

599 if (ctx->enc_keylen == 24)

600 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);

601 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {

602 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,

603 ctx->auth_keylen);

604 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */

605 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);

606 if (rc)

607 goto badkey;

608 }

609

610 /* STAT_PHASE_2: Create sequence */

611

612 switch (ctx->auth_mode) {

613 case DRV_HASH_SHA1:

614 case DRV_HASH_SHA256:

615 seq_len = hmac_setkey(desc, ctx);

616 break;

617 case DRV_HASH_XCBC_MAC:

618 seq_len = xcbc_setkey(desc, ctx);

619 break;

620 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */

621 break; /* No auth. key setup */

622 default:

623 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);

624 rc = -ENOTSUPP;

625 goto badkey;

626 }

627

628 /* STAT_PHASE_3: Submit sequence to HW */

629

630 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */

631 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);

632 if (rc) {

633 dev_err(dev, "send_request() failed (rc=%d)\n", rc);

634 goto setkey_error;

635 }

636 }

637

638 /* Update STAT_PHASE_3 */

639 return rc;

640

641 badkey:

642 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);

643

644 setkey_error:

645 return rc;

646 }

647

648 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,

649 unsigned int keylen)

650 {

651 struct crypto_authenc_keys keys;

652 int err;

653

654 err = crypto_authenc_extractkeys(&keys, key, keylen);

655 if (unlikely(err))

656 return err;

657

658 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:

659 cc_aead_setkey(aead, key, keylen);

660

661 memzero_explicit(&keys, sizeof(keys));

662 return err;

663 }

664

665 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,

666 unsigned int keylen)

667 {

668 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

669

670 if (keylen < 3)

671 return -EINVAL;

672

673 keylen -= 3;

674 memcpy(ctx->ctr_nonce, key + keylen, 3);

675

676 return cc_aead_setkey(tfm, key, keylen);

677 }

678

679 static int cc_aead_setauthsize(struct crypto_aead *authenc,

680 unsigned int authsize)

681 {

682 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);

683 struct device *dev = drvdata_to_dev(ctx->drvdata);

684

685 /* Unsupported auth. sizes */

686 if (authsize == 0 ||

687 authsize > crypto_aead_maxauthsize(authenc)) {

688 return -ENOTSUPP;

689 }

690

691 ctx->authsize = authsize;

692 dev_dbg(dev, "authlen=%d\n", ctx->authsize);

693

694 return 0;

695 }

696

697 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,

698 unsigned int authsize)

699 {

700 switch (authsize) {

701 case 8:

702 case 12:

703 case 16:

704 break;

705 default:

706 return -EINVAL;

707 }

708

709 return cc_aead_setauthsize(authenc, authsize);

710 }

711

712 static int cc_ccm_setauthsize(struct crypto_aead *authenc,

713 unsigned int authsize)

714 {

715 switch (authsize) {

716 case 4:

717 case 6:

718 case 8:

719 case 10:

720 case 12:

721 case 14:

722 case 16:

723 break;

724 default:

725 return -EINVAL;

726 }

727

728 return cc_aead_setauthsize(authenc, authsize);

729 }

730

731 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,

732 struct cc_hw_desc desc[], unsigned int *seq_size)

733 {

734 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);

735 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

736 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);

737 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;

738 unsigned int idx = *seq_size;

739 struct device *dev = drvdata_to_dev(ctx->drvdata);

740

741 switch (assoc_dma_type) {

742 case CC_DMA_BUF_DLLI:

743 dev_dbg(dev, "ASSOC buffer type DLLI\n");

744 hw_desc_init(&desc[idx]);

745 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),

746 areq_ctx->assoclen, NS_BIT);

747 set_flow_mode(&desc[idx], flow_mode);

748 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&

749 areq_ctx->cryptlen > 0)

750 set_din_not_last_indication(&desc[idx]);

751 break;

752 case CC_DMA_BUF_MLLI:

753 dev_dbg(dev, "ASSOC buffer type MLLI\n");

754 hw_desc_init(&desc[idx]);

755 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,

756 areq_ctx->assoc.mlli_nents, NS_BIT);

757 set_flow_mode(&desc[idx], flow_mode);

758 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&

759 areq_ctx->cryptlen > 0)

760 set_din_not_last_indication(&desc[idx]);

761 break;

762 case CC_DMA_BUF_NULL:

763 default:

764 dev_err(dev, "Invalid ASSOC buffer type\n");

765 }

766

767 *seq_size = (++idx);

768 }

769

770 static void cc_proc_authen_desc(struct aead_request *areq,

771 unsigned int flow_mode,

772 struct cc_hw_desc desc[],

773 unsigned int *seq_size, int direct)

774 {

775 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);

776 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;

777 unsigned int idx = *seq_size;

778 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);

779 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

780 struct device *dev = drvdata_to_dev(ctx->drvdata);

781

782 switch (data_dma_type) {

783 case CC_DMA_BUF_DLLI:

784 {

785 struct scatterlist *cipher =

786 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?

787 areq_ctx->dst_sgl : areq_ctx->src_sgl;

788

789 unsigned int offset =

790 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?

791 areq_ctx->dst_offset : areq_ctx->src_offset;

792 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");

793 hw_desc_init(&desc[idx]);

794 set_din_type(&desc[idx], DMA_DLLI,

795 (sg_dma_address(cipher) + offset),

796 areq_ctx->cryptlen, NS_BIT);

797 set_flow_mode(&desc[idx], flow_mode);

798 break;

799 }

800 case CC_DMA_BUF_MLLI:

801 {

802 /* DOUBLE-PASS flow (as default)

803 * assoc. + iv + data -compact in one table

804 * if assoclen is ZERO only IV perform

805 */

806 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;

807 u32 mlli_nents = areq_ctx->assoc.mlli_nents;

808

809 if (areq_ctx->is_single_pass) {

810 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {

811 mlli_addr = areq_ctx->dst.sram_addr;

812 mlli_nents = areq_ctx->dst.mlli_nents;

813 } else {

814 mlli_addr = areq_ctx->src.sram_addr;

815 mlli_nents = areq_ctx->src.mlli_nents;

816 }

817 }

818

819 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");

820 hw_desc_init(&desc[idx]);

821 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,

822 NS_BIT);

823 set_flow_mode(&desc[idx], flow_mode);

824 break;

825 }

826 case CC_DMA_BUF_NULL:

827 default:

828 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");

829 }

830

831 *seq_size = (++idx);

832 }

833

834 static void cc_proc_cipher_desc(struct aead_request *areq,

835 unsigned int flow_mode,

836 struct cc_hw_desc desc[],

837 unsigned int *seq_size)

838 {

839 unsigned int idx = *seq_size;

840 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);

841 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;

842 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);

843 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

844 struct device *dev = drvdata_to_dev(ctx->drvdata);

845

846 if (areq_ctx->cryptlen == 0)

847 return; /*null processing*/

848

849 switch (data_dma_type) {

850 case CC_DMA_BUF_DLLI:

851 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");

852 hw_desc_init(&desc[idx]);

853 set_din_type(&desc[idx], DMA_DLLI,

854 (sg_dma_address(areq_ctx->src_sgl) +

855 areq_ctx->src_offset), areq_ctx->cryptlen,

856 NS_BIT);

857 set_dout_dlli(&desc[idx],

858 (sg_dma_address(areq_ctx->dst_sgl) +

859 areq_ctx->dst_offset),

860 areq_ctx->cryptlen, NS_BIT, 0);

861 set_flow_mode(&desc[idx], flow_mode);

862 break;

863 case CC_DMA_BUF_MLLI:

864 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");

865 hw_desc_init(&desc[idx]);

866 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,

867 areq_ctx->src.mlli_nents, NS_BIT);

868 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,

869 areq_ctx->dst.mlli_nents, NS_BIT, 0);

870 set_flow_mode(&desc[idx], flow_mode);

871 break;

872 case CC_DMA_BUF_NULL:

873 default:

874 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");

875 }

876

877 *seq_size = (++idx);

878 }

879

880 static void cc_proc_digest_desc(struct aead_request *req,

881 struct cc_hw_desc desc[],

882 unsigned int *seq_size)

883 {

884 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

885 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

886 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

887 unsigned int idx = *seq_size;

888 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?

889 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;

890 int direct = req_ctx->gen_ctx.op_type;

891

892 /* Get final ICV result */

893 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {

894 hw_desc_init(&desc[idx]);

895 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

896 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

897 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,

898 NS_BIT, 1);

899 set_queue_last_ind(ctx->drvdata, &desc[idx]);

900 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {

901 set_aes_not_hash_mode(&desc[idx]);

902 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);

903 } else {

904 set_cipher_config0(&desc[idx],

905 HASH_DIGEST_RESULT_LITTLE_ENDIAN);

906 set_cipher_mode(&desc[idx], hash_mode);

907 }

908 } else { /*Decrypt*/

909 /* Get ICV out from hardware */

910 hw_desc_init(&desc[idx]);

911 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

912 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

913 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,

914 ctx->authsize, NS_BIT, 1);

915 set_queue_last_ind(ctx->drvdata, &desc[idx]);

916 set_cipher_config0(&desc[idx],

917 HASH_DIGEST_RESULT_LITTLE_ENDIAN);

918 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);

919 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {

920 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);

921 set_aes_not_hash_mode(&desc[idx]);

922 } else {

923 set_cipher_mode(&desc[idx], hash_mode);

924 }

925 }

926

927 *seq_size = (++idx);

928 }

929

930 static void cc_set_cipher_desc(struct aead_request *req,

931 struct cc_hw_desc desc[],

932 unsigned int *seq_size)

933 {

934 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

935 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

936 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

937 unsigned int hw_iv_size = req_ctx->hw_iv_size;

938 unsigned int idx = *seq_size;

939 int direct = req_ctx->gen_ctx.op_type;

940

941 /* Setup cipher state */

942 hw_desc_init(&desc[idx]);

943 set_cipher_config0(&desc[idx], direct);

944 set_flow_mode(&desc[idx], ctx->flow_mode);

945 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,

946 hw_iv_size, NS_BIT);

947 if (ctx->cipher_mode == DRV_CIPHER_CTR)

948 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

949 else

950 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

951 set_cipher_mode(&desc[idx], ctx->cipher_mode);

952 idx++;

953

954 /* Setup enc. key */

955 hw_desc_init(&desc[idx]);

956 set_cipher_config0(&desc[idx], direct);

957 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

958 set_flow_mode(&desc[idx], ctx->flow_mode);

959 if (ctx->flow_mode == S_DIN_to_AES) {

960 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,

961 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :

962 ctx->enc_keylen), NS_BIT);

963 set_key_size_aes(&desc[idx], ctx->enc_keylen);

964 } else {

965 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,

966 ctx->enc_keylen, NS_BIT);

967 set_key_size_des(&desc[idx], ctx->enc_keylen);

968 }

969 set_cipher_mode(&desc[idx], ctx->cipher_mode);

970 idx++;

971

972 *seq_size = idx;

973 }

974

975 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],

976 unsigned int *seq_size, unsigned int data_flow_mode)

977 {

978 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

979 int direct = req_ctx->gen_ctx.op_type;

980 unsigned int idx = *seq_size;

981

982 if (req_ctx->cryptlen == 0)

983 return; /*null processing*/

984

985 cc_set_cipher_desc(req, desc, &idx);

986 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);

987 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {

988 /* We must wait for DMA to write all cipher */

989 hw_desc_init(&desc[idx]);

990 set_din_no_dma(&desc[idx], 0, 0xfffff0);

991 set_dout_no_dma(&desc[idx], 0, 0, 1);

992 idx++;

993 }

994

995 *seq_size = idx;

996 }

997

998 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],

999 unsigned int *seq_size)

1000 {

1001 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1002 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1003 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?

1004 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;

1005 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?

1006 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;

1007 unsigned int idx = *seq_size;

1008

1009 /* Loading hash ipad xor key state */

1010 hw_desc_init(&desc[idx]);

1011 set_cipher_mode(&desc[idx], hash_mode);

1012 set_din_type(&desc[idx], DMA_DLLI,

1013 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,

1014 NS_BIT);

1015 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1016 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

1017 idx++;

1018

1019 /* Load init. digest len (64 bytes) */

1020 hw_desc_init(&desc[idx]);

1021 set_cipher_mode(&desc[idx], hash_mode);

1022 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),

1023 ctx->hash_len);

1024 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1025 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1026 idx++;

1027

1028 *seq_size = idx;

1029 }

1030

1031 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],

1032 unsigned int *seq_size)

1033 {

1034 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1035 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1036 unsigned int idx = *seq_size;

1037

1038 /* Loading MAC state */

1039 hw_desc_init(&desc[idx]);

1040 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);

1041 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

1042 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);

1043 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1044 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);

1045 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1046 set_aes_not_hash_mode(&desc[idx]);

1047 idx++;

1048

1049 /* Setup XCBC MAC K1 */

1050 hw_desc_init(&desc[idx]);

1051 set_din_type(&desc[idx], DMA_DLLI,

1052 ctx->auth_state.xcbc.xcbc_keys_dma_addr,

1053 AES_KEYSIZE_128, NS_BIT);

1054 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1055 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);

1056 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1057 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);

1058 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1059 set_aes_not_hash_mode(&desc[idx]);

1060 idx++;

1061

1062 /* Setup XCBC MAC K2 */

1063 hw_desc_init(&desc[idx]);

1064 set_din_type(&desc[idx], DMA_DLLI,

1065 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +

1066 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);

1067 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

1068 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);

1069 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1070 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);

1071 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1072 set_aes_not_hash_mode(&desc[idx]);

1073 idx++;

1074

1075 /* Setup XCBC MAC K3 */

1076 hw_desc_init(&desc[idx]);

1077 set_din_type(&desc[idx], DMA_DLLI,

1078 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +

1079 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);

1080 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);

1081 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);

1082 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1083 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);

1084 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1085 set_aes_not_hash_mode(&desc[idx]);

1086 idx++;

1087

1088 *seq_size = idx;

1089 }

1090

1091 static void cc_proc_header_desc(struct aead_request *req,

1092 struct cc_hw_desc desc[],

1093 unsigned int *seq_size)

1094 {

1095 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

1096 unsigned int idx = *seq_size;

1097

1098 /* Hash associated data */

1099 if (areq_ctx->assoclen > 0)

1100 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);

1101

1102 /* Hash IV */

1103 *seq_size = idx;

1104 }

1105

1106 static void cc_proc_scheme_desc(struct aead_request *req,

1107 struct cc_hw_desc desc[],

1108 unsigned int *seq_size)

1109 {

1110 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1111 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1112 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;

1113 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?

1114 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;

1115 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?

1116 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;

1117 unsigned int idx = *seq_size;

1118

1119 hw_desc_init(&desc[idx]);

1120 set_cipher_mode(&desc[idx], hash_mode);

1121 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,

1122 ctx->hash_len);

1123 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

1124 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);

1125 set_cipher_do(&desc[idx], DO_PAD);

1126 idx++;

1127

1128 /* Get final ICV result */

1129 hw_desc_init(&desc[idx]);

1130 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,

1131 digest_size);

1132 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

1133 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

1134 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);

1135 set_cipher_mode(&desc[idx], hash_mode);

1136 idx++;

1137

1138 /* Loading hash opad xor key state */

1139 hw_desc_init(&desc[idx]);

1140 set_cipher_mode(&desc[idx], hash_mode);

1141 set_din_type(&desc[idx], DMA_DLLI,

1142 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),

1143 digest_size, NS_BIT);

1144 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1145 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

1146 idx++;

1147

1148 /* Load init. digest len (64 bytes) */

1149 hw_desc_init(&desc[idx]);

1150 set_cipher_mode(&desc[idx], hash_mode);

1151 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),

1152 ctx->hash_len);

1153 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);

1154 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1155 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1156 idx++;

1157

1158 /* Perform HASH update */

1159 hw_desc_init(&desc[idx]);

1160 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,

1161 digest_size);

1162 set_flow_mode(&desc[idx], DIN_HASH);

1163 idx++;

1164

1165 *seq_size = idx;

1166 }

1167

1168 static void cc_mlli_to_sram(struct aead_request *req,

1169 struct cc_hw_desc desc[], unsigned int *seq_size)

1170 {

1171 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1172 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1173 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1174 struct device *dev = drvdata_to_dev(ctx->drvdata);

1175

1176 if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||

1177 req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||

1178 !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) {

1179 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",

1180 (unsigned int)ctx->drvdata->mlli_sram_addr,

1181 req_ctx->mlli_params.mlli_len);

1182 /* Copy MLLI table host-to-sram */

1183 hw_desc_init(&desc[*seq_size]);

1184 set_din_type(&desc[*seq_size], DMA_DLLI,

1185 req_ctx->mlli_params.mlli_dma_addr,

1186 req_ctx->mlli_params.mlli_len, NS_BIT);

1187 set_dout_sram(&desc[*seq_size],

1188 ctx->drvdata->mlli_sram_addr,

1189 req_ctx->mlli_params.mlli_len);

1190 set_flow_mode(&desc[*seq_size], BYPASS);

1191 (*seq_size)++;

1192 }

1193 }

1194

1195 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,

1196 enum cc_flow_mode setup_flow_mode,

1197 bool is_single_pass)

1198 {

1199 enum cc_flow_mode data_flow_mode;

1200

1201 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {

1202 if (setup_flow_mode == S_DIN_to_AES)

1203 data_flow_mode = is_single_pass ?

1204 AES_to_HASH_and_DOUT : DIN_AES_DOUT;

1205 else

1206 data_flow_mode = is_single_pass ?

1207 DES_to_HASH_and_DOUT : DIN_DES_DOUT;

1208 } else { /* Decrypt */

1209 if (setup_flow_mode == S_DIN_to_AES)

1210 data_flow_mode = is_single_pass ?

1211 AES_and_HASH : DIN_AES_DOUT;

1212 else

1213 data_flow_mode = is_single_pass ?

1214 DES_and_HASH : DIN_DES_DOUT;

1215 }

1216

1217 return data_flow_mode;

1218 }

1219

1220 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],

1221 unsigned int *seq_size)

1222 {

1223 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1224 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1225 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1226 int direct = req_ctx->gen_ctx.op_type;

1227 unsigned int data_flow_mode =

1228 cc_get_data_flow(direct, ctx->flow_mode,

1229 req_ctx->is_single_pass);

1230

1231 if (req_ctx->is_single_pass) {

1232 /**

1233 * Single-pass flow

1234 */

1235 cc_set_hmac_desc(req, desc, seq_size);

1236 cc_set_cipher_desc(req, desc, seq_size);

1237 cc_proc_header_desc(req, desc, seq_size);

1238 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);

1239 cc_proc_scheme_desc(req, desc, seq_size);

1240 cc_proc_digest_desc(req, desc, seq_size);

1241 return;

1242 }

1243

1244 /**

1245 * Double-pass flow

1246 * Fallback for unsupported single-pass modes,

1247 * i.e. using assoc. data of non-word-multiple

1248 */

1249 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {

1250 /* encrypt first.. */

1251 cc_proc_cipher(req, desc, seq_size, data_flow_mode);

1252 /* authenc after..*/

1253 cc_set_hmac_desc(req, desc, seq_size);

1254 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);

1255 cc_proc_scheme_desc(req, desc, seq_size);

1256 cc_proc_digest_desc(req, desc, seq_size);

1257

1258 } else { /*DECRYPT*/

1259 /* authenc first..*/

1260 cc_set_hmac_desc(req, desc, seq_size);

1261 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);

1262 cc_proc_scheme_desc(req, desc, seq_size);

1263 /* decrypt after.. */

1264 cc_proc_cipher(req, desc, seq_size, data_flow_mode);

1265 /* read the digest result with setting the completion bit

1266 * must be after the cipher operation

1267 */

1268 cc_proc_digest_desc(req, desc, seq_size);

1269 }

1270 }

1271

1272 static void

1273 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],

1274 unsigned int *seq_size)

1275 {

1276 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1277 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1278 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1279 int direct = req_ctx->gen_ctx.op_type;

1280 unsigned int data_flow_mode =

1281 cc_get_data_flow(direct, ctx->flow_mode,

1282 req_ctx->is_single_pass);

1283

1284 if (req_ctx->is_single_pass) {

1285 /**

1286 * Single-pass flow

1287 */

1288 cc_set_xcbc_desc(req, desc, seq_size);

1289 cc_set_cipher_desc(req, desc, seq_size);

1290 cc_proc_header_desc(req, desc, seq_size);

1291 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);

1292 cc_proc_digest_desc(req, desc, seq_size);

1293 return;

1294 }

1295

1296 /**

1297 * Double-pass flow

1298 * Fallback for unsupported single-pass modes,

1299 * i.e. using assoc. data of non-word-multiple

1300 */

1301 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {

1302 /* encrypt first.. */

1303 cc_proc_cipher(req, desc, seq_size, data_flow_mode);

1304 /* authenc after.. */

1305 cc_set_xcbc_desc(req, desc, seq_size);

1306 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);

1307 cc_proc_digest_desc(req, desc, seq_size);

1308 } else { /*DECRYPT*/

1309 /* authenc first.. */

1310 cc_set_xcbc_desc(req, desc, seq_size);

1311 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);

1312 /* decrypt after..*/

1313 cc_proc_cipher(req, desc, seq_size, data_flow_mode);

1314 /* read the digest result with setting the completion bit

1315 * must be after the cipher operation

1316 */

1317 cc_proc_digest_desc(req, desc, seq_size);

1318 }

1319 }

1320

1321 static int validate_data_size(struct cc_aead_ctx *ctx,

1322 enum drv_crypto_direction direct,

1323 struct aead_request *req)

1324 {

1325 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

1326 struct device *dev = drvdata_to_dev(ctx->drvdata);

1327 unsigned int assoclen = areq_ctx->assoclen;

1328 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?

1329 (req->cryptlen - ctx->authsize) : req->cryptlen;

1330

1331 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&

1332 req->cryptlen < ctx->authsize)

1333 goto data_size_err;

1334

1335 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/

1336

1337 switch (ctx->flow_mode) {

1338 case S_DIN_to_AES:

1339 if (ctx->cipher_mode == DRV_CIPHER_CBC &&

1340 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))

1341 goto data_size_err;

1342 if (ctx->cipher_mode == DRV_CIPHER_CCM)

1343 break;

1344 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {

1345 if (areq_ctx->plaintext_authenticate_only)

1346 areq_ctx->is_single_pass = false;

1347 break;

1348 }

1349

1350 if (!IS_ALIGNED(assoclen, sizeof(u32)))

1351 areq_ctx->is_single_pass = false;

1352

1353 if (ctx->cipher_mode == DRV_CIPHER_CTR &&

1354 !IS_ALIGNED(cipherlen, sizeof(u32)))

1355 areq_ctx->is_single_pass = false;

1356

1357 break;

1358 case S_DIN_to_DES:

1359 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))

1360 goto data_size_err;

1361 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))

1362 areq_ctx->is_single_pass = false;

1363 break;

1364 default:

1365 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);

1366 goto data_size_err;

1367 }

1368

1369 return 0;

1370

1371 data_size_err:

1372 return -EINVAL;

1373 }

1374

1375 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)

1376 {

1377 unsigned int len = 0;

1378

1379 if (header_size == 0)

1380 return 0;

1381

1382 if (header_size < ((1UL << 16) - (1UL << 8))) {

1383 len = 2;

1384

1385 pa0_buff[0] = (header_size >> 8) & 0xFF;

1386 pa0_buff[1] = header_size & 0xFF;

1387 } else {

1388 len = 6;

1389

1390 pa0_buff[0] = 0xFF;

1391 pa0_buff[1] = 0xFE;

1392 pa0_buff[2] = (header_size >> 24) & 0xFF;

1393 pa0_buff[3] = (header_size >> 16) & 0xFF;

1394 pa0_buff[4] = (header_size >> 8) & 0xFF;

1395 pa0_buff[5] = header_size & 0xFF;

1396 }

1397

1398 return len;

1399 }

1400

1401 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)

1402 {

1403 __be32 data;

1404

1405 memset(block, 0, csize);

1406 block += csize;

1407

1408 if (csize >= 4)

1409 csize = 4;

1410 else if (msglen > (1 << (8 * csize)))

1411 return -EOVERFLOW;

1412

1413 data = cpu_to_be32(msglen);

1414 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);

1415

1416 return 0;

1417 }

1418

1419 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],

1420 unsigned int *seq_size)

1421 {

1422 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1423 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1424 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1425 unsigned int idx = *seq_size;

1426 unsigned int cipher_flow_mode;

1427 dma_addr_t mac_result;

1428

1429 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {

1430 cipher_flow_mode = AES_to_HASH_and_DOUT;

1431 mac_result = req_ctx->mac_buf_dma_addr;

1432 } else { /* Encrypt */

1433 cipher_flow_mode = AES_and_HASH;

1434 mac_result = req_ctx->icv_dma_addr;

1435 }

1436

1437 /* load key */

1438 hw_desc_init(&desc[idx]);

1439 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);

1440 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,

1441 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :

1442 ctx->enc_keylen), NS_BIT);

1443 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1444 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1445 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1446 set_flow_mode(&desc[idx], S_DIN_to_AES);

1447 idx++;

1448

1449 /* load ctr state */

1450 hw_desc_init(&desc[idx]);

1451 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);

1452 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1453 set_din_type(&desc[idx], DMA_DLLI,

1454 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);

1455 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1456 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

1457 set_flow_mode(&desc[idx], S_DIN_to_AES);

1458 idx++;

1459

1460 /* load MAC key */

1461 hw_desc_init(&desc[idx]);

1462 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);

1463 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,

1464 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :

1465 ctx->enc_keylen), NS_BIT);

1466 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1467 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1468 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1469 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1470 set_aes_not_hash_mode(&desc[idx]);

1471 idx++;

1472

1473 /* load MAC state */

1474 hw_desc_init(&desc[idx]);

1475 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);

1476 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1477 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,

1478 AES_BLOCK_SIZE, NS_BIT);

1479 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);

1480 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

1481 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1482 set_aes_not_hash_mode(&desc[idx]);

1483 idx++;

1484

1485 /* process assoc data */

1486 if (req_ctx->assoclen > 0) {

1487 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);

1488 } else {

1489 hw_desc_init(&desc[idx]);

1490 set_din_type(&desc[idx], DMA_DLLI,

1491 sg_dma_address(&req_ctx->ccm_adata_sg),

1492 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);

1493 set_flow_mode(&desc[idx], DIN_HASH);

1494 idx++;

1495 }

1496

1497 /* process the cipher */

1498 if (req_ctx->cryptlen)

1499 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);

1500

1501 /* Read temporal MAC */

1502 hw_desc_init(&desc[idx]);

1503 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);

1504 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,

1505 NS_BIT, 0);

1506 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

1507 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);

1508 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

1509 set_aes_not_hash_mode(&desc[idx]);

1510 idx++;

1511

1512 /* load AES-CTR state (for last MAC calculation)*/

1513 hw_desc_init(&desc[idx]);

1514 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);

1515 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);

1516 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,

1517 AES_BLOCK_SIZE, NS_BIT);

1518 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1519 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

1520 set_flow_mode(&desc[idx], S_DIN_to_AES);

1521 idx++;

1522

1523 hw_desc_init(&desc[idx]);

1524 set_din_no_dma(&desc[idx], 0, 0xfffff0);

1525 set_dout_no_dma(&desc[idx], 0, 0, 1);

1526 idx++;

1527

1528 /* encrypt the "T" value and store MAC in mac_state */

1529 hw_desc_init(&desc[idx]);

1530 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,

1531 ctx->authsize, NS_BIT);

1532 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);

1533 set_queue_last_ind(ctx->drvdata, &desc[idx]);

1534 set_flow_mode(&desc[idx], DIN_AES_DOUT);

1535 idx++;

1536

1537 *seq_size = idx;

1538 return 0;

1539 }

1540

1541 static int config_ccm_adata(struct aead_request *req)

1542 {

1543 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1544 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1545 struct device *dev = drvdata_to_dev(ctx->drvdata);

1546 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1547 //unsigned int size_of_a = 0, rem_a_size = 0;

1548 unsigned int lp = req->iv[0];

1549 /* Note: The code assume that req->iv[0] already contains the value

1550 * of L' of RFC3610

1551 */

1552 unsigned int l = lp + 1; /* This is L' of RFC 3610. */

1553 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */

1554 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;

1555 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;

1556 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;

1557 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==

1558 DRV_CRYPTO_DIRECTION_ENCRYPT) ?

1559 req->cryptlen :

1560 (req->cryptlen - ctx->authsize);

1561 int rc;

1562

1563 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);

1564 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);

1565

1566 /* taken from crypto/ccm.c */

1567 /* 2 <= L <= 8, so 1 <= L' <= 7. */

1568 if (l < 2 || l > 8) {

1569 dev_err(dev, "illegal iv value %X\n", req->iv[0]);

1570 return -EINVAL;

1571 }

1572 memcpy(b0, req->iv, AES_BLOCK_SIZE);

1573

1574 /* format control info per RFC 3610 and

1575 * NIST Special Publication 800-38C

1576 */

1577 *b0 |= (8 * ((m - 2) / 2));

1578 if (req_ctx->assoclen > 0)

1579 *b0 |= 64; /* Enable bit 6 if Adata exists. */

1580

1581 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */

1582 if (rc) {

1583 dev_err(dev, "message len overflow detected");

1584 return rc;

1585 }

1586 /* END of "taken from crypto/ccm.c" */

1587

1588 /* l(a) - size of associated data. */

1589 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen);

1590

1591 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);

1592 req->iv[15] = 1;

1593

1594 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);

1595 ctr_count_0[15] = 0;

1596

1597 return 0;

1598 }

1599

1600 static void cc_proc_rfc4309_ccm(struct aead_request *req)

1601 {

1602 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1603 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1604 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

1605

1606 /* L' */

1607 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);

1608 /* For RFC 4309, always use 4 bytes for message length

1609 * (at most 2^32-1 bytes).

1610 */

1611 areq_ctx->ctr_iv[0] = 3;

1612

1613 /* In RFC 4309 there is an 11-bytes nonce+IV part,

1614 * that we build here.

1615 */

1616 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,

1617 CCM_BLOCK_NONCE_SIZE);

1618 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,

1619 CCM_BLOCK_IV_SIZE);

1620 req->iv = areq_ctx->ctr_iv;

1621 areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE;

1622 }

1623

1624 static void cc_set_ghash_desc(struct aead_request *req,

1625 struct cc_hw_desc desc[], unsigned int *seq_size)

1626 {

1627 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1628 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1629 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1630 unsigned int idx = *seq_size;

1631

1632 /* load key to AES*/

1633 hw_desc_init(&desc[idx]);

1634 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);

1635 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);

1636 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,

1637 ctx->enc_keylen, NS_BIT);

1638 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1639 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1640 set_flow_mode(&desc[idx], S_DIN_to_AES);

1641 idx++;

1642

1643 /* process one zero block to generate hkey */

1644 hw_desc_init(&desc[idx]);

1645 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);

1646 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,

1647 NS_BIT, 0);

1648 set_flow_mode(&desc[idx], DIN_AES_DOUT);

1649 idx++;

1650

1651 /* Memory Barrier */

1652 hw_desc_init(&desc[idx]);

1653 set_din_no_dma(&desc[idx], 0, 0xfffff0);

1654 set_dout_no_dma(&desc[idx], 0, 0, 1);

1655 idx++;

1656

1657 /* Load GHASH subkey */

1658 hw_desc_init(&desc[idx]);

1659 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,

1660 AES_BLOCK_SIZE, NS_BIT);

1661 set_dout_no_dma(&desc[idx], 0, 0, 1);

1662 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1663 set_aes_not_hash_mode(&desc[idx]);

1664 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);

1665 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);

1666 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1667 idx++;

1668

1669 /* Configure Hash Engine to work with GHASH.

1670 * Since it was not possible to extend HASH submodes to add GHASH,

1671 * The following command is necessary in order to

1672 * select GHASH (according to HW designers)

1673 */

1674 hw_desc_init(&desc[idx]);

1675 set_din_no_dma(&desc[idx], 0, 0xfffff0);

1676 set_dout_no_dma(&desc[idx], 0, 0, 1);

1677 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1678 set_aes_not_hash_mode(&desc[idx]);

1679 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);

1680 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK

1681 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);

1682 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);

1683 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1684 idx++;

1685

1686 /* Load GHASH initial STATE (which is 0). (for any hash there is an

1687 * initial state)

1688 */

1689 hw_desc_init(&desc[idx]);

1690 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);

1691 set_dout_no_dma(&desc[idx], 0, 0, 1);

1692 set_flow_mode(&desc[idx], S_DIN_to_HASH);

1693 set_aes_not_hash_mode(&desc[idx]);

1694 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);

1695 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);

1696 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);

1697 idx++;

1698

1699 *seq_size = idx;

1700 }

1701

1702 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],

1703 unsigned int *seq_size)

1704 {

1705 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1706 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1707 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1708 unsigned int idx = *seq_size;

1709

1710 /* load key to AES*/

1711 hw_desc_init(&desc[idx]);

1712 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);

1713 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);

1714 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,

1715 ctx->enc_keylen, NS_BIT);

1716 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1717 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);

1718 set_flow_mode(&desc[idx], S_DIN_to_AES);

1719 idx++;

1720

1721 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {

1722 /* load AES/CTR initial CTR value inc by 2*/

1723 hw_desc_init(&desc[idx]);

1724 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);

1725 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1726 set_din_type(&desc[idx], DMA_DLLI,

1727 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,

1728 NS_BIT);

1729 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);

1730 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

1731 set_flow_mode(&desc[idx], S_DIN_to_AES);

1732 idx++;

1733 }

1734

1735 *seq_size = idx;

1736 }

1737

1738 static void cc_proc_gcm_result(struct aead_request *req,

1739 struct cc_hw_desc desc[],

1740 unsigned int *seq_size)

1741 {

1742 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1743 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1744 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1745 dma_addr_t mac_result;

1746 unsigned int idx = *seq_size;

1747

1748 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {

1749 mac_result = req_ctx->mac_buf_dma_addr;

1750 } else { /* Encrypt */

1751 mac_result = req_ctx->icv_dma_addr;

1752 }

1753

1754 /* process(ghash) gcm_block_len */

1755 hw_desc_init(&desc[idx]);

1756 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,

1757 AES_BLOCK_SIZE, NS_BIT);

1758 set_flow_mode(&desc[idx], DIN_HASH);

1759 idx++;

1760

1761 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */

1762 hw_desc_init(&desc[idx]);

1763 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);

1764 set_din_no_dma(&desc[idx], 0, 0xfffff0);

1765 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,

1766 NS_BIT, 0);

1767 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);

1768 set_flow_mode(&desc[idx], S_HASH_to_DOUT);

1769 set_aes_not_hash_mode(&desc[idx]);

1770

1771 idx++;

1772

1773 /* load AES/CTR initial CTR value inc by 1*/

1774 hw_desc_init(&desc[idx]);

1775 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);

1776 set_key_size_aes(&desc[idx], ctx->enc_keylen);

1777 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,

1778 AES_BLOCK_SIZE, NS_BIT);

1779 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);

1780 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);

1781 set_flow_mode(&desc[idx], S_DIN_to_AES);

1782 idx++;

1783

1784 /* Memory Barrier */

1785 hw_desc_init(&desc[idx]);

1786 set_din_no_dma(&desc[idx], 0, 0xfffff0);

1787 set_dout_no_dma(&desc[idx], 0, 0, 1);

1788 idx++;

1789

1790 /* process GCTR on stored GHASH and store MAC in mac_state*/

1791 hw_desc_init(&desc[idx]);

1792 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);

1793 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,

1794 AES_BLOCK_SIZE, NS_BIT);

1795 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);

1796 set_queue_last_ind(ctx->drvdata, &desc[idx]);

1797 set_flow_mode(&desc[idx], DIN_AES_DOUT);

1798 idx++;

1799

1800 *seq_size = idx;

1801 }

1802

1803 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],

1804 unsigned int *seq_size)

1805 {

1806 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1807 unsigned int cipher_flow_mode;

1808

1809 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {

1810 cipher_flow_mode = AES_and_HASH;

1811 } else { /* Encrypt */

1812 cipher_flow_mode = AES_to_HASH_and_DOUT;

1813 }

1814

1815 //in RFC4543 no data to encrypt. just copy data from src to dest.

1816 if (req_ctx->plaintext_authenticate_only) {

1817 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);

1818 cc_set_ghash_desc(req, desc, seq_size);

1819 /* process(ghash) assoc data */

1820 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);

1821 cc_set_gctr_desc(req, desc, seq_size);

1822 cc_proc_gcm_result(req, desc, seq_size);

1823 return 0;

1824 }

1825

1826 // for gcm and rfc4106.

1827 cc_set_ghash_desc(req, desc, seq_size);

1828 /* process(ghash) assoc data */

1829 if (req_ctx->assoclen > 0)

1830 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);

1831 cc_set_gctr_desc(req, desc, seq_size);

1832 /* process(gctr+ghash) */

1833 if (req_ctx->cryptlen)

1834 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);

1835 cc_proc_gcm_result(req, desc, seq_size);

1836

1837 return 0;

1838 }

1839

1840 static int config_gcm_context(struct aead_request *req)

1841 {

1842 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1843 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1844 struct aead_req_ctx *req_ctx = aead_request_ctx(req);

1845 struct device *dev = drvdata_to_dev(ctx->drvdata);

1846

1847 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==

1848 DRV_CRYPTO_DIRECTION_ENCRYPT) ?

1849 req->cryptlen :

1850 (req->cryptlen - ctx->authsize);

1851 __be32 counter = cpu_to_be32(2);

1852

1853 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n",

1854 __func__, cryptlen, req_ctx->assoclen, ctx->authsize);

1855

1856 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);

1857

1858 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);

1859

1860 memcpy(req->iv + 12, &counter, 4);

1861 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);

1862

1863 counter = cpu_to_be32(1);

1864 memcpy(req->iv + 12, &counter, 4);

1865 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);

1866

1867 if (!req_ctx->plaintext_authenticate_only) {

1868 __be64 temp64;

1869

1870 temp64 = cpu_to_be64(req_ctx->assoclen * 8);

1871 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));

1872 temp64 = cpu_to_be64(cryptlen * 8);

1873 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);

1874 } else {

1875 /* rfc4543=> all data(AAD,IV,Plain) are considered additional

1876 * data that is nothing is encrypted.

1877 */

1878 __be64 temp64;

1879

1880 temp64 = cpu_to_be64((req_ctx->assoclen +

1881 GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8);

1882 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));

1883 temp64 = 0;

1884 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);

1885 }

1886

1887 return 0;

1888 }

1889

1890 static void cc_proc_rfc4_gcm(struct aead_request *req)

1891 {

1892 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1893 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1894 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

1895

1896 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,

1897 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);

1898 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,

1899 GCM_BLOCK_RFC4_IV_SIZE);

1900 req->iv = areq_ctx->ctr_iv;

1901 areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;

1902 }

1903

1904 static int cc_proc_aead(struct aead_request *req,

1905 enum drv_crypto_direction direct)

1906 {

1907 int rc = 0;

1908 int seq_len = 0;

1909 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];

1910 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

1911 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

1912 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

1913 struct device *dev = drvdata_to_dev(ctx->drvdata);

1914 struct cc_crypto_req cc_req = {};

1915

1916 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",

1917 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),

1918 ctx, req, req->iv, sg_virt(req->src), req->src->offset,

1919 sg_virt(req->dst), req->dst->offset, req->cryptlen);

1920

1921 /* STAT_PHASE_0: Init and sanity checks */

1922

1923 /* Check data length according to mode */

1924 if (validate_data_size(ctx, direct, req)) {

1925 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",

1926 req->cryptlen, areq_ctx->assoclen);

1927 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);

1928 return -EINVAL;

1929 }

1930

1931 /* Setup request structure */

1932 cc_req.user_cb = (void *)cc_aead_complete;

1933 cc_req.user_arg = (void *)req;

1934

1935 /* Setup request context */

1936 areq_ctx->gen_ctx.op_type = direct;

1937 areq_ctx->req_authsize = ctx->authsize;

1938 areq_ctx->cipher_mode = ctx->cipher_mode;

1939

1940 /* STAT_PHASE_1: Map buffers */

1941

1942 if (ctx->cipher_mode == DRV_CIPHER_CTR) {

1943 /* Build CTR IV - Copy nonce from last 4 bytes in

1944 * CTR key to first 4 bytes in CTR IV

1945 */

1946 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,

1947 CTR_RFC3686_NONCE_SIZE);

1948 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv,

1949 CTR_RFC3686_IV_SIZE);

1950 /* Initialize counter portion of counter block */

1951 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +

1952 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);

1953

1954 /* Replace with counter iv */

1955 req->iv = areq_ctx->ctr_iv;

1956 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;

1957 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||

1958 (ctx->cipher_mode == DRV_CIPHER_GCTR)) {

1959 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;

1960 if (areq_ctx->ctr_iv != req->iv) {

1961 memcpy(areq_ctx->ctr_iv, req->iv,

1962 crypto_aead_ivsize(tfm));

1963 req->iv = areq_ctx->ctr_iv;

1964 }

1965 } else {

1966 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);

1967 }

1968

1969 if (ctx->cipher_mode == DRV_CIPHER_CCM) {

1970 rc = config_ccm_adata(req);

1971 if (rc) {

1972 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",

1973 rc);

1974 goto exit;

1975 }

1976 } else {

1977 areq_ctx->ccm_hdr_size = ccm_header_size_null;

1978 }

1979

1980 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {

1981 rc = config_gcm_context(req);

1982 if (rc) {

1983 dev_dbg(dev, "config_gcm_context() returned with a failure %d!",

1984 rc);

1985 goto exit;

1986 }

1987 }

1988

1989 rc = cc_map_aead_request(ctx->drvdata, req);

1990 if (rc) {

1991 dev_err(dev, "map_request() failed\n");

1992 goto exit;

1993 }

1994

1995 /* STAT_PHASE_2: Create sequence */

1996

1997 /* Load MLLI tables to SRAM if necessary */

1998 cc_mlli_to_sram(req, desc, &seq_len);

1999

2000 /*TODO: move seq len by reference */

2001 switch (ctx->auth_mode) {

2002 case DRV_HASH_SHA1:

2003 case DRV_HASH_SHA256:

2004 cc_hmac_authenc(req, desc, &seq_len);

2005 break;

2006 case DRV_HASH_XCBC_MAC:

2007 cc_xcbc_authenc(req, desc, &seq_len);

2008 break;

2009 case DRV_HASH_NULL:

2010 if (ctx->cipher_mode == DRV_CIPHER_CCM)

2011 cc_ccm(req, desc, &seq_len);

2012 if (ctx->cipher_mode == DRV_CIPHER_GCTR)

2013 cc_gcm(req, desc, &seq_len);

2014 break;

2015 default:

2016 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);

2017 cc_unmap_aead_request(dev, req);

2018 rc = -ENOTSUPP;

2019 goto exit;

2020 }

2021

2022 /* STAT_PHASE_3: Lock HW and push sequence */

2023

2024 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);

2025

2026 if (rc != -EINPROGRESS && rc != -EBUSY) {

2027 dev_err(dev, "send_request() failed (rc=%d)\n", rc);

2028 cc_unmap_aead_request(dev, req);

2029 }

2030

2031 exit:

2032 return rc;

2033 }

2034

2035 static int cc_aead_encrypt(struct aead_request *req)

2036 {

2037 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2038 int rc;

2039

2040 memset(areq_ctx, 0, sizeof(*areq_ctx));

2041

2042 /* No generated IV required */

2043 areq_ctx->backup_iv = req->iv;

2044 areq_ctx->assoclen = req->assoclen;

2045 areq_ctx->is_gcm4543 = false;

2046

2047 areq_ctx->plaintext_authenticate_only = false;

2048

2049 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);

2050 if (rc != -EINPROGRESS && rc != -EBUSY)

2051 req->iv = areq_ctx->backup_iv;

2052

2053 return rc;

2054 }

2055

2056 static int cc_rfc4309_ccm_encrypt(struct aead_request *req)

2057 {

2058 /* Very similar to cc_aead_encrypt() above. */

2059

2060 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2061 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

2062 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2063 struct device *dev = drvdata_to_dev(ctx->drvdata);

2064 int rc = -EINVAL;

2065

2066 if (!valid_assoclen(req)) {

2067 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);

2068 goto out;

2069 }

2070

2071 memset(areq_ctx, 0, sizeof(*areq_ctx));

2072

2073 /* No generated IV required */

2074 areq_ctx->backup_iv = req->iv;

2075 areq_ctx->assoclen = req->assoclen;

2076 areq_ctx->is_gcm4543 = true;

2077

2078 cc_proc_rfc4309_ccm(req);

2079

2080 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);

2081 if (rc != -EINPROGRESS && rc != -EBUSY)

2082 req->iv = areq_ctx->backup_iv;

2083 out:

2084 return rc;

2085 }

2086

2087 static int cc_aead_decrypt(struct aead_request *req)

2088 {

2089 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2090 int rc;

2091

2092 memset(areq_ctx, 0, sizeof(*areq_ctx));

2093

2094 /* No generated IV required */

2095 areq_ctx->backup_iv = req->iv;

2096 areq_ctx->assoclen = req->assoclen;

2097 areq_ctx->is_gcm4543 = false;

2098

2099 areq_ctx->plaintext_authenticate_only = false;

2100

2101 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);

2102 if (rc != -EINPROGRESS && rc != -EBUSY)

2103 req->iv = areq_ctx->backup_iv;

2104

2105 return rc;

2106 }

2107

2108 static int cc_rfc4309_ccm_decrypt(struct aead_request *req)

2109 {

2110 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

2111 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2112 struct device *dev = drvdata_to_dev(ctx->drvdata);

2113 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2114 int rc = -EINVAL;

2115

2116 if (!valid_assoclen(req)) {

2117 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);

2118 goto out;

2119 }

2120

2121 memset(areq_ctx, 0, sizeof(*areq_ctx));

2122

2123 /* No generated IV required */

2124 areq_ctx->backup_iv = req->iv;

2125 areq_ctx->assoclen = req->assoclen;

2126

2127 areq_ctx->is_gcm4543 = true;

2128 cc_proc_rfc4309_ccm(req);

2129

2130 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);

2131 if (rc != -EINPROGRESS && rc != -EBUSY)

2132 req->iv = areq_ctx->backup_iv;

2133

2134 out:

2135 return rc;

2136 }

2137

2138 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,

2139 unsigned int keylen)

2140 {

2141 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2142 struct device *dev = drvdata_to_dev(ctx->drvdata);

2143

2144 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);

2145

2146 if (keylen < 4)

2147 return -EINVAL;

2148

2149 keylen -= 4;

2150 memcpy(ctx->ctr_nonce, key + keylen, 4);

2151

2152 return cc_aead_setkey(tfm, key, keylen);

2153 }

2154

2155 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,

2156 unsigned int keylen)

2157 {

2158 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2159 struct device *dev = drvdata_to_dev(ctx->drvdata);

2160

2161 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);

2162

2163 if (keylen < 4)

2164 return -EINVAL;

2165

2166 keylen -= 4;

2167 memcpy(ctx->ctr_nonce, key + keylen, 4);

2168

2169 return cc_aead_setkey(tfm, key, keylen);

2170 }

2171

2172 static int cc_gcm_setauthsize(struct crypto_aead *authenc,

2173 unsigned int authsize)

2174 {

2175 switch (authsize) {

2176 case 4:

2177 case 8:

2178 case 12:

2179 case 13:

2180 case 14:

2181 case 15:

2182 case 16:

2183 break;

2184 default:

2185 return -EINVAL;

2186 }

2187

2188 return cc_aead_setauthsize(authenc, authsize);

2189 }

2190

2191 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,

2192 unsigned int authsize)

2193 {

2194 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);

2195 struct device *dev = drvdata_to_dev(ctx->drvdata);

2196

2197 dev_dbg(dev, "authsize %d\n", authsize);

2198

2199 switch (authsize) {

2200 case 8:

2201 case 12:

2202 case 16:

2203 break;

2204 default:

2205 return -EINVAL;

2206 }

2207

2208 return cc_aead_setauthsize(authenc, authsize);

2209 }

2210

2211 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,

2212 unsigned int authsize)

2213 {

2214 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);

2215 struct device *dev = drvdata_to_dev(ctx->drvdata);

2216

2217 dev_dbg(dev, "authsize %d\n", authsize);

2218

2219 if (authsize != 16)

2220 return -EINVAL;

2221

2222 return cc_aead_setauthsize(authenc, authsize);

2223 }

2224

2225 static int cc_rfc4106_gcm_encrypt(struct aead_request *req)

2226 {

2227 /* Very similar to cc_aead_encrypt() above. */

2228

2229 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

2230 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2231 struct device *dev = drvdata_to_dev(ctx->drvdata);

2232 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2233 int rc = -EINVAL;

2234

2235 if (!valid_assoclen(req)) {

2236 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);

2237 goto out;

2238 }

2239

2240 memset(areq_ctx, 0, sizeof(*areq_ctx));

2241

2242 /* No generated IV required */

2243 areq_ctx->backup_iv = req->iv;

2244 areq_ctx->assoclen = req->assoclen;

2245 areq_ctx->plaintext_authenticate_only = false;

2246

2247 cc_proc_rfc4_gcm(req);

2248 areq_ctx->is_gcm4543 = true;

2249

2250 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);

2251 if (rc != -EINPROGRESS && rc != -EBUSY)

2252 req->iv = areq_ctx->backup_iv;

2253 out:

2254 return rc;

2255 }

2256

2257 static int cc_rfc4543_gcm_encrypt(struct aead_request *req)

2258 {

2259 /* Very similar to cc_aead_encrypt() above. */

2260 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

2261 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2262 struct device *dev = drvdata_to_dev(ctx->drvdata);

2263 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2264 int rc = -EINVAL;

2265

2266 if (!valid_assoclen(req)) {

2267 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);

2268 goto out;

2269 }

2270

2271 memset(areq_ctx, 0, sizeof(*areq_ctx));

2272

2273 //plaintext is not encryped with rfc4543

2274 areq_ctx->plaintext_authenticate_only = true;

2275

2276 /* No generated IV required */

2277 areq_ctx->backup_iv = req->iv;

2278 areq_ctx->assoclen = req->assoclen;

2279

2280 cc_proc_rfc4_gcm(req);

2281 areq_ctx->is_gcm4543 = true;

2282

2283 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);

2284 if (rc != -EINPROGRESS && rc != -EBUSY)

2285 req->iv = areq_ctx->backup_iv;

2286 out:

2287 return rc;

2288 }

2289

2290 static int cc_rfc4106_gcm_decrypt(struct aead_request *req)

2291 {

2292 /* Very similar to cc_aead_decrypt() above. */

2293

2294 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

2295 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2296 struct device *dev = drvdata_to_dev(ctx->drvdata);

2297 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2298 int rc = -EINVAL;

2299

2300 if (!valid_assoclen(req)) {

2301 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);

2302 goto out;

2303 }

2304

2305 memset(areq_ctx, 0, sizeof(*areq_ctx));

2306

2307 /* No generated IV required */

2308 areq_ctx->backup_iv = req->iv;

2309 areq_ctx->assoclen = req->assoclen;

2310 areq_ctx->plaintext_authenticate_only = false;

2311

2312 cc_proc_rfc4_gcm(req);

2313 areq_ctx->is_gcm4543 = true;

2314

2315 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);

2316 if (rc != -EINPROGRESS && rc != -EBUSY)

2317 req->iv = areq_ctx->backup_iv;

2318 out:

2319 return rc;

2320 }

2321

2322 static int cc_rfc4543_gcm_decrypt(struct aead_request *req)

2323 {

2324 /* Very similar to cc_aead_decrypt() above. */

2325 struct crypto_aead *tfm = crypto_aead_reqtfm(req);

2326 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);

2327 struct device *dev = drvdata_to_dev(ctx->drvdata);

2328 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);

2329 int rc = -EINVAL;

2330

2331 if (!valid_assoclen(req)) {

2332 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);

2333 goto out;

2334 }

2335

2336 memset(areq_ctx, 0, sizeof(*areq_ctx));

2337

2338 //plaintext is not decryped with rfc4543

2339 areq_ctx->plaintext_authenticate_only = true;

2340

2341 /* No generated IV required */

2342 areq_ctx->backup_iv = req->iv;

2343 areq_ctx->assoclen = req->assoclen;

2344

2345 cc_proc_rfc4_gcm(req);

2346 areq_ctx->is_gcm4543 = true;

2347

2348 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);

2349 if (rc != -EINPROGRESS && rc != -EBUSY)

2350 req->iv = areq_ctx->backup_iv;

2351 out:

2352 return rc;

2353 }

2354

2355 /* aead alg */

2356 static struct cc_alg_template aead_algs[] = {

2357 {

2358 .name = "authenc(hmac(sha1),cbc(aes))",

2359 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",

2360 .blocksize = AES_BLOCK_SIZE,

2361 .template_aead = {

2362 .setkey = cc_aead_setkey,

2363 .setauthsize = cc_aead_setauthsize,

2364 .encrypt = cc_aead_encrypt,

2365 .decrypt = cc_aead_decrypt,

2366 .init = cc_aead_init,

2367 .exit = cc_aead_exit,

2368 .ivsize = AES_BLOCK_SIZE,

2369 .maxauthsize = SHA1_DIGEST_SIZE,

2370 },

2371 .cipher_mode = DRV_CIPHER_CBC,

2372 .flow_mode = S_DIN_to_AES,

2373 .auth_mode = DRV_HASH_SHA1,

2374 .min_hw_rev = CC_HW_REV_630,

2375 .std_body = CC_STD_NIST,

2376 },

2377 {

2378 .name = "authenc(hmac(sha1),cbc(des3_ede))",

2379 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",

2380 .blocksize = DES3_EDE_BLOCK_SIZE,

2381 .template_aead = {

2382 .setkey = cc_des3_aead_setkey,

2383 .setauthsize = cc_aead_setauthsize,

2384 .encrypt = cc_aead_encrypt,

2385 .decrypt = cc_aead_decrypt,

2386 .init = cc_aead_init,

2387 .exit = cc_aead_exit,

2388 .ivsize = DES3_EDE_BLOCK_SIZE,

2389 .maxauthsize = SHA1_DIGEST_SIZE,

2390 },

2391 .cipher_mode = DRV_CIPHER_CBC,

2392 .flow_mode = S_DIN_to_DES,

2393 .auth_mode = DRV_HASH_SHA1,

2394 .min_hw_rev = CC_HW_REV_630,

2395 .std_body = CC_STD_NIST,

2396 },

2397 {

2398 .name = "authenc(hmac(sha256),cbc(aes))",

2399 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",

2400 .blocksize = AES_BLOCK_SIZE,

2401 .template_aead = {

2402 .setkey = cc_aead_setkey,

2403 .setauthsize = cc_aead_setauthsize,

2404 .encrypt = cc_aead_encrypt,

2405 .decrypt = cc_aead_decrypt,

2406 .init = cc_aead_init,

2407 .exit = cc_aead_exit,

2408 .ivsize = AES_BLOCK_SIZE,

2409 .maxauthsize = SHA256_DIGEST_SIZE,

2410 },

2411 .cipher_mode = DRV_CIPHER_CBC,

2412 .flow_mode = S_DIN_to_AES,

2413 .auth_mode = DRV_HASH_SHA256,

2414 .min_hw_rev = CC_HW_REV_630,

2415 .std_body = CC_STD_NIST,

2416 },

2417 {

2418 .name = "authenc(hmac(sha256),cbc(des3_ede))",

2419 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",

2420 .blocksize = DES3_EDE_BLOCK_SIZE,

2421 .template_aead = {

2422 .setkey = cc_des3_aead_setkey,

2423 .setauthsize = cc_aead_setauthsize,

2424 .encrypt = cc_aead_encrypt,

2425 .decrypt = cc_aead_decrypt,

2426 .init = cc_aead_init,

2427 .exit = cc_aead_exit,

2428 .ivsize = DES3_EDE_BLOCK_SIZE,

2429 .maxauthsize = SHA256_DIGEST_SIZE,

2430 },

2431 .cipher_mode = DRV_CIPHER_CBC,

2432 .flow_mode = S_DIN_to_DES,

2433 .auth_mode = DRV_HASH_SHA256,

2434 .min_hw_rev = CC_HW_REV_630,

2435 .std_body = CC_STD_NIST,

2436 },

2437 {

2438 .name = "authenc(xcbc(aes),cbc(aes))",

2439 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",

2440 .blocksize = AES_BLOCK_SIZE,

2441 .template_aead = {

2442 .setkey = cc_aead_setkey,

2443 .setauthsize = cc_aead_setauthsize,

2444 .encrypt = cc_aead_encrypt,

2445 .decrypt = cc_aead_decrypt,

2446 .init = cc_aead_init,

2447 .exit = cc_aead_exit,

2448 .ivsize = AES_BLOCK_SIZE,

2449 .maxauthsize = AES_BLOCK_SIZE,

2450 },

2451 .cipher_mode = DRV_CIPHER_CBC,

2452 .flow_mode = S_DIN_to_AES,

2453 .auth_mode = DRV_HASH_XCBC_MAC,

2454 .min_hw_rev = CC_HW_REV_630,

2455 .std_body = CC_STD_NIST,

2456 },

2457 {

2458 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",

2459 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",

2460 .blocksize = 1,

2461 .template_aead = {

2462 .setkey = cc_aead_setkey,

2463 .setauthsize = cc_aead_setauthsize,

2464 .encrypt = cc_aead_encrypt,

2465 .decrypt = cc_aead_decrypt,

2466 .init = cc_aead_init,

2467 .exit = cc_aead_exit,

2468 .ivsize = CTR_RFC3686_IV_SIZE,

2469 .maxauthsize = SHA1_DIGEST_SIZE,

2470 },

2471 .cipher_mode = DRV_CIPHER_CTR,

2472 .flow_mode = S_DIN_to_AES,

2473 .auth_mode = DRV_HASH_SHA1,

2474 .min_hw_rev = CC_HW_REV_630,

2475 .std_body = CC_STD_NIST,

2476 },

2477 {

2478 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",

2479 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",

2480 .blocksize = 1,

2481 .template_aead = {

2482 .setkey = cc_aead_setkey,

2483 .setauthsize = cc_aead_setauthsize,

2484 .encrypt = cc_aead_encrypt,

2485 .decrypt = cc_aead_decrypt,

2486 .init = cc_aead_init,

2487 .exit = cc_aead_exit,

2488 .ivsize = CTR_RFC3686_IV_SIZE,

2489 .maxauthsize = SHA256_DIGEST_SIZE,

2490 },

2491 .cipher_mode = DRV_CIPHER_CTR,

2492 .flow_mode = S_DIN_to_AES,

2493 .auth_mode = DRV_HASH_SHA256,

2494 .min_hw_rev = CC_HW_REV_630,

2495 .std_body = CC_STD_NIST,

2496 },

2497 {

2498 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",

2499 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",

2500 .blocksize = 1,

2501 .template_aead = {

2502 .setkey = cc_aead_setkey,

2503 .setauthsize = cc_aead_setauthsize,

2504 .encrypt = cc_aead_encrypt,

2505 .decrypt = cc_aead_decrypt,

2506 .init = cc_aead_init,

2507 .exit = cc_aead_exit,

2508 .ivsize = CTR_RFC3686_IV_SIZE,

2509 .maxauthsize = AES_BLOCK_SIZE,

2510 },

2511 .cipher_mode = DRV_CIPHER_CTR,

2512 .flow_mode = S_DIN_to_AES,

2513 .auth_mode = DRV_HASH_XCBC_MAC,

2514 .min_hw_rev = CC_HW_REV_630,

2515 .std_body = CC_STD_NIST,

2516 },

2517 {

2518 .name = "ccm(aes)",

2519 .driver_name = "ccm-aes-ccree",

2520 .blocksize = 1,

2521 .template_aead = {

2522 .setkey = cc_aead_setkey,

2523 .setauthsize = cc_ccm_setauthsize,

2524 .encrypt = cc_aead_encrypt,

2525 .decrypt = cc_aead_decrypt,

2526 .init = cc_aead_init,

2527 .exit = cc_aead_exit,

2528 .ivsize = AES_BLOCK_SIZE,

2529 .maxauthsize = AES_BLOCK_SIZE,

2530 },

2531 .cipher_mode = DRV_CIPHER_CCM,

2532 .flow_mode = S_DIN_to_AES,

2533 .auth_mode = DRV_HASH_NULL,

2534 .min_hw_rev = CC_HW_REV_630,

2535 .std_body = CC_STD_NIST,

2536 },

2537 {

2538 .name = "rfc4309(ccm(aes))",

2539 .driver_name = "rfc4309-ccm-aes-ccree",

2540 .blocksize = 1,

2541 .template_aead = {

2542 .setkey = cc_rfc4309_ccm_setkey,

2543 .setauthsize = cc_rfc4309_ccm_setauthsize,

2544 .encrypt = cc_rfc4309_ccm_encrypt,

2545 .decrypt = cc_rfc4309_ccm_decrypt,

2546 .init = cc_aead_init,

2547 .exit = cc_aead_exit,

2548 .ivsize = CCM_BLOCK_IV_SIZE,

2549 .maxauthsize = AES_BLOCK_SIZE,

2550 },

2551 .cipher_mode = DRV_CIPHER_CCM,

2552 .flow_mode = S_DIN_to_AES,

2553 .auth_mode = DRV_HASH_NULL,

2554 .min_hw_rev = CC_HW_REV_630,

2555 .std_body = CC_STD_NIST,

2556 },

2557 {

2558 .name = "gcm(aes)",

2559 .driver_name = "gcm-aes-ccree",

2560 .blocksize = 1,

2561 .template_aead = {

2562 .setkey = cc_aead_setkey,

2563 .setauthsize = cc_gcm_setauthsize,

2564 .encrypt = cc_aead_encrypt,

2565 .decrypt = cc_aead_decrypt,

2566 .init = cc_aead_init,

2567 .exit = cc_aead_exit,

2568 .ivsize = 12,

2569 .maxauthsize = AES_BLOCK_SIZE,

2570 },

2571 .cipher_mode = DRV_CIPHER_GCTR,

2572 .flow_mode = S_DIN_to_AES,

2573 .auth_mode = DRV_HASH_NULL,

2574 .min_hw_rev = CC_HW_REV_630,

2575 .std_body = CC_STD_NIST,

2576 },

2577 {

2578 .name = "rfc4106(gcm(aes))",

2579 .driver_name = "rfc4106-gcm-aes-ccree",

2580 .blocksize = 1,

2581 .template_aead = {

2582 .setkey = cc_rfc4106_gcm_setkey,

2583 .setauthsize = cc_rfc4106_gcm_setauthsize,

2584 .encrypt = cc_rfc4106_gcm_encrypt,

2585 .decrypt = cc_rfc4106_gcm_decrypt,

2586 .init = cc_aead_init,

2587 .exit = cc_aead_exit,

2588 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,

2589 .maxauthsize = AES_BLOCK_SIZE,

2590 },

2591 .cipher_mode = DRV_CIPHER_GCTR,

2592 .flow_mode = S_DIN_to_AES,

2593 .auth_mode = DRV_HASH_NULL,

2594 .min_hw_rev = CC_HW_REV_630,

2595 .std_body = CC_STD_NIST,

2596 },

2597 {

2598 .name = "rfc4543(gcm(aes))",

2599 .driver_name = "rfc4543-gcm-aes-ccree",

2600 .blocksize = 1,

2601 .template_aead = {

2602 .setkey = cc_rfc4543_gcm_setkey,

2603 .setauthsize = cc_rfc4543_gcm_setauthsize,

2604 .encrypt = cc_rfc4543_gcm_encrypt,

2605 .decrypt = cc_rfc4543_gcm_decrypt,

2606 .init = cc_aead_init,

2607 .exit = cc_aead_exit,

2608 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,

2609 .maxauthsize = AES_BLOCK_SIZE,

2610 },

2611 .cipher_mode = DRV_CIPHER_GCTR,

2612 .flow_mode = S_DIN_to_AES,

2613 .auth_mode = DRV_HASH_NULL,

2614 .min_hw_rev = CC_HW_REV_630,

2615 .std_body = CC_STD_NIST,

2616 },

2617 };

2618

2619 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,

2620 struct device *dev)

2621 {

2622 struct cc_crypto_alg *t_alg;

2623 struct aead_alg *alg;

2624

2625 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);

2626 if (!t_alg)

2627 return ERR_PTR(-ENOMEM);

2628

2629 alg = &tmpl->template_aead;

2630

2631 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);

2632 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",

2633 tmpl->driver_name);

2634 alg->base.cra_module = THIS_MODULE;

2635 alg->base.cra_priority = CC_CRA_PRIO;

2636

2637 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);

2638 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;

2639 alg->init = cc_aead_init;

2640 alg->exit = cc_aead_exit;

2641

2642 t_alg->aead_alg = *alg;

2643

2644 t_alg->cipher_mode = tmpl->cipher_mode;

2645 t_alg->flow_mode = tmpl->flow_mode;

2646 t_alg->auth_mode = tmpl->auth_mode;

2647

2648 return t_alg;

2649 }

2650

2651 int cc_aead_free(struct cc_drvdata *drvdata)

2652 {

2653 struct cc_crypto_alg *t_alg, *n;

2654 struct cc_aead_handle *aead_handle =

2655 (struct cc_aead_handle *)drvdata->aead_handle;

2656

2657 if (aead_handle) {

2658 /* Remove registered algs */

2659 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,

2660 entry) {

2661 crypto_unregister_aead(&t_alg->aead_alg);

2662 list_del(&t_alg->entry);

2663 kfree(t_alg);

2664 }

2665 kfree(aead_handle);

2666 drvdata->aead_handle = NULL;

2667 }

2668

2669 return 0;

2670 }

2671

2672 int cc_aead_alloc(struct cc_drvdata *drvdata)

2673 {

2674 struct cc_aead_handle *aead_handle;

2675 struct cc_crypto_alg *t_alg;

2676 int rc = -ENOMEM;

2677 int alg;

2678 struct device *dev = drvdata_to_dev(drvdata);

2679

2680 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);

2681 if (!aead_handle) {

2682 rc = -ENOMEM;

2683 goto fail0;

2684 }

2685

2686 INIT_LIST_HEAD(&aead_handle->aead_list);

2687 drvdata->aead_handle = aead_handle;

2688

2689 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,

2690 MAX_HMAC_DIGEST_SIZE);

2691

2692 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {

2693 dev_err(dev, "SRAM pool exhausted\n");

2694 rc = -ENOMEM;

2695 goto fail1;

2696 }

2697

2698 /* Linux crypto */

2699 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {

2700 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) ||

2701 !(drvdata->std_bodies & aead_algs[alg].std_body))

2702 continue;

2703

2704 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);

2705 if (IS_ERR(t_alg)) {

2706 rc = PTR_ERR(t_alg);

2707 dev_err(dev, "%s alg allocation failed\n",

2708 aead_algs[alg].driver_name);

2709 goto fail1;

2710 }

2711 t_alg->drvdata = drvdata;

2712 rc = crypto_register_aead(&t_alg->aead_alg);

2713 if (rc) {

2714 dev_err(dev, "%s alg registration failed\n",

2715 t_alg->aead_alg.base.cra_driver_name);

2716 goto fail2;

2717 } else {

2718 list_add_tail(&t_alg->entry, &aead_handle->aead_list);

2719 dev_dbg(dev, "Registered %s\n",

2720 t_alg->aead_alg.base.cra_driver_name);

2721 }

2722 }

2723

2724 return 0;

2725

2726 fail2:

2727 kfree(t_alg);

2728 fail1:

2729 cc_aead_free(drvdata);

2730 fail0:

2731 return rc;

2732 }