]> git.ipfire.org Git - people/ms/linux.git/blob - drivers/crypto/talitos.c
projects / people / ms / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'ext4_for_linus_stable' of git://git.kernel.org/pub/scm/linux/kernel/git...
[people/ms/linux.git] / drivers / crypto / talitos.c
1 /*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_platform.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/io.h>
40 #include <linux/spinlock.h>
41 #include <linux/rtnetlink.h>
42 #include <linux/slab.h>
43
44 #include <crypto/algapi.h>
45 #include <crypto/aes.h>
46 #include <crypto/des.h>
47 #include <crypto/sha.h>
48 #include <crypto/md5.h>
49 #include <crypto/internal/aead.h>
50 #include <crypto/authenc.h>
51 #include <crypto/skcipher.h>
52 #include <crypto/hash.h>
53 #include <crypto/internal/hash.h>
54 #include <crypto/scatterwalk.h>
55
56 #include "talitos.h"
57
58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
59 bool is_sec1)
60 {
61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
62 if (!is_sec1)
63 ptr->eptr = upper_32_bits(dma_addr);
64 }
65
66 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
67 bool is_sec1)
68 {
69 if (is_sec1) {
70 ptr->res = 0;
71 ptr->len1 = cpu_to_be16(len);
72 } else {
73 ptr->len = cpu_to_be16(len);
74 }
75 }
76
77 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
78 bool is_sec1)
79 {
80 if (is_sec1)
81 return be16_to_cpu(ptr->len1);
82 else
83 return be16_to_cpu(ptr->len);
84 }
85
86 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
87 {
88 if (!is_sec1)
89 ptr->j_extent = 0;
90 }
91
92 /*
93 * map virtual single (contiguous) pointer to h/w descriptor pointer
94 */
95 static void map_single_talitos_ptr(struct device *dev,
96 struct talitos_ptr *ptr,
97 unsigned int len, void *data,
98 enum dma_data_direction dir)
99 {
100 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
101 struct talitos_private *priv = dev_get_drvdata(dev);
102 bool is_sec1 = has_ftr_sec1(priv);
103
104 to_talitos_ptr_len(ptr, len, is_sec1);
105 to_talitos_ptr(ptr, dma_addr, is_sec1);
106 to_talitos_ptr_extent_clear(ptr, is_sec1);
107 }
108
109 /*
110 * unmap bus single (contiguous) h/w descriptor pointer
111 */
112 static void unmap_single_talitos_ptr(struct device *dev,
113 struct talitos_ptr *ptr,
114 enum dma_data_direction dir)
115 {
116 struct talitos_private *priv = dev_get_drvdata(dev);
117 bool is_sec1 = has_ftr_sec1(priv);
118
119 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
120 from_talitos_ptr_len(ptr, is_sec1), dir);
121 }
122
123 static int reset_channel(struct device *dev, int ch)
124 {
125 struct talitos_private *priv = dev_get_drvdata(dev);
126 unsigned int timeout = TALITOS_TIMEOUT;
127 bool is_sec1 = has_ftr_sec1(priv);
128
129 if (is_sec1) {
130 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
131 TALITOS1_CCCR_LO_RESET);
132
133 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
134 TALITOS1_CCCR_LO_RESET) && --timeout)
135 cpu_relax();
136 } else {
137 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
138 TALITOS2_CCCR_RESET);
139
140 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
141 TALITOS2_CCCR_RESET) && --timeout)
142 cpu_relax();
143 }
144
145 if (timeout == 0) {
146 dev_err(dev, "failed to reset channel %d\n", ch);
147 return -EIO;
148 }
149
150 /* set 36-bit addressing, done writeback enable and done IRQ enable */
151 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
152 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
153
154 /* and ICCR writeback, if available */
155 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
156 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
157 TALITOS_CCCR_LO_IWSE);
158
159 return 0;
160 }
161
162 static int reset_device(struct device *dev)
163 {
164 struct talitos_private *priv = dev_get_drvdata(dev);
165 unsigned int timeout = TALITOS_TIMEOUT;
166 bool is_sec1 = has_ftr_sec1(priv);
167 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
168
169 setbits32(priv->reg + TALITOS_MCR, mcr);
170
171 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
172 && --timeout)
173 cpu_relax();
174
175 if (priv->irq[1]) {
176 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
177 setbits32(priv->reg + TALITOS_MCR, mcr);
178 }
179
180 if (timeout == 0) {
181 dev_err(dev, "failed to reset device\n");
182 return -EIO;
183 }
184
185 return 0;
186 }
187
188 /*
189 * Reset and initialize the device
190 */
191 static int init_device(struct device *dev)
192 {
193 struct talitos_private *priv = dev_get_drvdata(dev);
194 int ch, err;
195 bool is_sec1 = has_ftr_sec1(priv);
196
197 /*
198 * Master reset
199 * errata documentation: warning: certain SEC interrupts
200 * are not fully cleared by writing the MCR:SWR bit,
201 * set bit twice to completely reset
202 */
203 err = reset_device(dev);
204 if (err)
205 return err;
206
207 err = reset_device(dev);
208 if (err)
209 return err;
210
211 /* reset channels */
212 for (ch = 0; ch < priv->num_channels; ch++) {
213 err = reset_channel(dev, ch);
214 if (err)
215 return err;
216 }
217
218 /* enable channel done and error interrupts */
219 if (is_sec1) {
220 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
221 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
222 /* disable parity error check in DEU (erroneous? test vect.) */
223 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
224 } else {
225 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
226 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
227 }
228
229 /* disable integrity check error interrupts (use writeback instead) */
230 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
231 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
232 TALITOS_MDEUICR_LO_ICE);
233
234 return 0;
235 }
236
237 /**
238 * talitos_submit - submits a descriptor to the device for processing
239 * @dev: the SEC device to be used
240 * @ch: the SEC device channel to be used
241 * @desc: the descriptor to be processed by the device
242 * @callback: whom to call when processing is complete
243 * @context: a handle for use by caller (optional)
244 *
245 * desc must contain valid dma-mapped (bus physical) address pointers.
246 * callback must check err and feedback in descriptor header
247 * for device processing status.
248 */
249 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
250 void (*callback)(struct device *dev,
251 struct talitos_desc *desc,
252 void *context, int error),
253 void *context)
254 {
255 struct talitos_private *priv = dev_get_drvdata(dev);
256 struct talitos_request *request;
257 unsigned long flags;
258 int head;
259 bool is_sec1 = has_ftr_sec1(priv);
260
261 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
262
263 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
264 /* h/w fifo is full */
265 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
266 return -EAGAIN;
267 }
268
269 head = priv->chan[ch].head;
270 request = &priv->chan[ch].fifo[head];
271
272 /* map descriptor and save caller data */
273 if (is_sec1) {
274 desc->hdr1 = desc->hdr;
275 desc->next_desc = 0;
276 request->dma_desc = dma_map_single(dev, &desc->hdr1,
277 TALITOS_DESC_SIZE,
278 DMA_BIDIRECTIONAL);
279 } else {
280 request->dma_desc = dma_map_single(dev, desc,
281 TALITOS_DESC_SIZE,
282 DMA_BIDIRECTIONAL);
283 }
284 request->callback = callback;
285 request->context = context;
286
287 /* increment fifo head */
288 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
289
290 smp_wmb();
291 request->desc = desc;
292
293 /* GO! */
294 wmb();
295 out_be32(priv->chan[ch].reg + TALITOS_FF,
296 upper_32_bits(request->dma_desc));
297 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
298 lower_32_bits(request->dma_desc));
299
300 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
301
302 return -EINPROGRESS;
303 }
304 EXPORT_SYMBOL(talitos_submit);
305
306 /*
307 * process what was done, notify callback of error if not
308 */
309 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
310 {
311 struct talitos_private *priv = dev_get_drvdata(dev);
312 struct talitos_request *request, saved_req;
313 unsigned long flags;
314 int tail, status;
315 bool is_sec1 = has_ftr_sec1(priv);
316
317 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
318
319 tail = priv->chan[ch].tail;
320 while (priv->chan[ch].fifo[tail].desc) {
321 __be32 hdr;
322
323 request = &priv->chan[ch].fifo[tail];
324
325 /* descriptors with their done bits set don't get the error */
326 rmb();
327 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
328
329 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
330 status = 0;
331 else
332 if (!error)
333 break;
334 else
335 status = error;
336
337 dma_unmap_single(dev, request->dma_desc,
338 TALITOS_DESC_SIZE,
339 DMA_BIDIRECTIONAL);
340
341 /* copy entries so we can call callback outside lock */
342 saved_req.desc = request->desc;
343 saved_req.callback = request->callback;
344 saved_req.context = request->context;
345
346 /* release request entry in fifo */
347 smp_wmb();
348 request->desc = NULL;
349
350 /* increment fifo tail */
351 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
352
353 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
354
355 atomic_dec(&priv->chan[ch].submit_count);
356
357 saved_req.callback(dev, saved_req.desc, saved_req.context,
358 status);
359 /* channel may resume processing in single desc error case */
360 if (error && !reset_ch && status == error)
361 return;
362 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
363 tail = priv->chan[ch].tail;
364 }
365
366 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
367 }
368
369 /*
370 * process completed requests for channels that have done status
371 */
372 #define DEF_TALITOS1_DONE(name, ch_done_mask) \
373 static void talitos1_done_##name(unsigned long data) \
374 { \
375 struct device *dev = (struct device *)data; \
376 struct talitos_private *priv = dev_get_drvdata(dev); \
377 unsigned long flags; \
378 \
379 if (ch_done_mask & 0x10000000) \
380 flush_channel(dev, 0, 0, 0); \
381 if (priv->num_channels == 1) \
382 goto out; \
383 if (ch_done_mask & 0x40000000) \
384 flush_channel(dev, 1, 0, 0); \
385 if (ch_done_mask & 0x00010000) \
386 flush_channel(dev, 2, 0, 0); \
387 if (ch_done_mask & 0x00040000) \
388 flush_channel(dev, 3, 0, 0); \
389 \
390 out: \
391 /* At this point, all completed channels have been processed */ \
392 /* Unmask done interrupts for channels completed later on. */ \
393 spin_lock_irqsave(&priv->reg_lock, flags); \
394 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
395 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
396 spin_unlock_irqrestore(&priv->reg_lock, flags); \
397 }
398
399 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
400
401 #define DEF_TALITOS2_DONE(name, ch_done_mask) \
402 static void talitos2_done_##name(unsigned long data) \
403 { \
404 struct device *dev = (struct device *)data; \
405 struct talitos_private *priv = dev_get_drvdata(dev); \
406 unsigned long flags; \
407 \
408 if (ch_done_mask & 1) \
409 flush_channel(dev, 0, 0, 0); \
410 if (priv->num_channels == 1) \
411 goto out; \
412 if (ch_done_mask & (1 << 2)) \
413 flush_channel(dev, 1, 0, 0); \
414 if (ch_done_mask & (1 << 4)) \
415 flush_channel(dev, 2, 0, 0); \
416 if (ch_done_mask & (1 << 6)) \
417 flush_channel(dev, 3, 0, 0); \
418 \
419 out: \
420 /* At this point, all completed channels have been processed */ \
421 /* Unmask done interrupts for channels completed later on. */ \
422 spin_lock_irqsave(&priv->reg_lock, flags); \
423 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
424 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
425 spin_unlock_irqrestore(&priv->reg_lock, flags); \
426 }
427
428 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
429 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
430 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
431
432 /*
433 * locate current (offending) descriptor
434 */
435 static u32 current_desc_hdr(struct device *dev, int ch)
436 {
437 struct talitos_private *priv = dev_get_drvdata(dev);
438 int tail, iter;
439 dma_addr_t cur_desc;
440
441 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
442 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
443
444 if (!cur_desc) {
445 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
446 return 0;
447 }
448
449 tail = priv->chan[ch].tail;
450
451 iter = tail;
452 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
453 iter = (iter + 1) & (priv->fifo_len - 1);
454 if (iter == tail) {
455 dev_err(dev, "couldn't locate current descriptor\n");
456 return 0;
457 }
458 }
459
460 return priv->chan[ch].fifo[iter].desc->hdr;
461 }
462
463 /*
464 * user diagnostics; report root cause of error based on execution unit status
465 */
466 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
467 {
468 struct talitos_private *priv = dev_get_drvdata(dev);
469 int i;
470
471 if (!desc_hdr)
472 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
473
474 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
475 case DESC_HDR_SEL0_AFEU:
476 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
477 in_be32(priv->reg_afeu + TALITOS_EUISR),
478 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
479 break;
480 case DESC_HDR_SEL0_DEU:
481 dev_err(dev, "DEUISR 0x%08x_%08x\n",
482 in_be32(priv->reg_deu + TALITOS_EUISR),
483 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
484 break;
485 case DESC_HDR_SEL0_MDEUA:
486 case DESC_HDR_SEL0_MDEUB:
487 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
488 in_be32(priv->reg_mdeu + TALITOS_EUISR),
489 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
490 break;
491 case DESC_HDR_SEL0_RNG:
492 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
493 in_be32(priv->reg_rngu + TALITOS_ISR),
494 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
495 break;
496 case DESC_HDR_SEL0_PKEU:
497 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
498 in_be32(priv->reg_pkeu + TALITOS_EUISR),
499 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
500 break;
501 case DESC_HDR_SEL0_AESU:
502 dev_err(dev, "AESUISR 0x%08x_%08x\n",
503 in_be32(priv->reg_aesu + TALITOS_EUISR),
504 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
505 break;
506 case DESC_HDR_SEL0_CRCU:
507 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
508 in_be32(priv->reg_crcu + TALITOS_EUISR),
509 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
510 break;
511 case DESC_HDR_SEL0_KEU:
512 dev_err(dev, "KEUISR 0x%08x_%08x\n",
513 in_be32(priv->reg_pkeu + TALITOS_EUISR),
514 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
515 break;
516 }
517
518 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
519 case DESC_HDR_SEL1_MDEUA:
520 case DESC_HDR_SEL1_MDEUB:
521 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
522 in_be32(priv->reg_mdeu + TALITOS_EUISR),
523 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
524 break;
525 case DESC_HDR_SEL1_CRCU:
526 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
527 in_be32(priv->reg_crcu + TALITOS_EUISR),
528 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
529 break;
530 }
531
532 for (i = 0; i < 8; i++)
533 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
534 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
535 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
536 }
537
538 /*
539 * recover from error interrupts
540 */
541 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
542 {
543 struct talitos_private *priv = dev_get_drvdata(dev);
544 unsigned int timeout = TALITOS_TIMEOUT;
545 int ch, error, reset_dev = 0;
546 u32 v_lo;
547 bool is_sec1 = has_ftr_sec1(priv);
548 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
549
550 for (ch = 0; ch < priv->num_channels; ch++) {
551 /* skip channels without errors */
552 if (is_sec1) {
553 /* bits 29, 31, 17, 19 */
554 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
555 continue;
556 } else {
557 if (!(isr & (1 << (ch * 2 + 1))))
558 continue;
559 }
560
561 error = -EINVAL;
562
563 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
564
565 if (v_lo & TALITOS_CCPSR_LO_DOF) {
566 dev_err(dev, "double fetch fifo overflow error\n");
567 error = -EAGAIN;
568 reset_ch = 1;
569 }
570 if (v_lo & TALITOS_CCPSR_LO_SOF) {
571 /* h/w dropped descriptor */
572 dev_err(dev, "single fetch fifo overflow error\n");
573 error = -EAGAIN;
574 }
575 if (v_lo & TALITOS_CCPSR_LO_MDTE)
576 dev_err(dev, "master data transfer error\n");
577 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
578 dev_err(dev, is_sec1 ? "pointeur not complete error\n"
579 : "s/g data length zero error\n");
580 if (v_lo & TALITOS_CCPSR_LO_FPZ)
581 dev_err(dev, is_sec1 ? "parity error\n"
582 : "fetch pointer zero error\n");
583 if (v_lo & TALITOS_CCPSR_LO_IDH)
584 dev_err(dev, "illegal descriptor header error\n");
585 if (v_lo & TALITOS_CCPSR_LO_IEU)
586 dev_err(dev, is_sec1 ? "static assignment error\n"
587 : "invalid exec unit error\n");
588 if (v_lo & TALITOS_CCPSR_LO_EU)
589 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
590 if (!is_sec1) {
591 if (v_lo & TALITOS_CCPSR_LO_GB)
592 dev_err(dev, "gather boundary error\n");
593 if (v_lo & TALITOS_CCPSR_LO_GRL)
594 dev_err(dev, "gather return/length error\n");
595 if (v_lo & TALITOS_CCPSR_LO_SB)
596 dev_err(dev, "scatter boundary error\n");
597 if (v_lo & TALITOS_CCPSR_LO_SRL)
598 dev_err(dev, "scatter return/length error\n");
599 }
600
601 flush_channel(dev, ch, error, reset_ch);
602
603 if (reset_ch) {
604 reset_channel(dev, ch);
605 } else {
606 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
607 TALITOS2_CCCR_CONT);
608 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
609 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
610 TALITOS2_CCCR_CONT) && --timeout)
611 cpu_relax();
612 if (timeout == 0) {
613 dev_err(dev, "failed to restart channel %d\n",
614 ch);
615 reset_dev = 1;
616 }
617 }
618 }
619 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
620 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
621 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
622 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
623 isr, isr_lo);
624 else
625 dev_err(dev, "done overflow, internal time out, or "
626 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
627
628 /* purge request queues */
629 for (ch = 0; ch < priv->num_channels; ch++)
630 flush_channel(dev, ch, -EIO, 1);
631
632 /* reset and reinitialize the device */
633 init_device(dev);
634 }
635 }
636
637 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
638 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
639 { \
640 struct device *dev = data; \
641 struct talitos_private *priv = dev_get_drvdata(dev); \
642 u32 isr, isr_lo; \
643 unsigned long flags; \
644 \
645 spin_lock_irqsave(&priv->reg_lock, flags); \
646 isr = in_be32(priv->reg + TALITOS_ISR); \
647 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
648 /* Acknowledge interrupt */ \
649 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
650 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
651 \
652 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
653 spin_unlock_irqrestore(&priv->reg_lock, flags); \
654 talitos_error(dev, isr & ch_err_mask, isr_lo); \
655 } \
656 else { \
657 if (likely(isr & ch_done_mask)) { \
658 /* mask further done interrupts. */ \
659 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
660 /* done_task will unmask done interrupts at exit */ \
661 tasklet_schedule(&priv->done_task[tlet]); \
662 } \
663 spin_unlock_irqrestore(&priv->reg_lock, flags); \
664 } \
665 \
666 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
667 IRQ_NONE; \
668 }
669
670 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
671
672 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
673 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
674 { \
675 struct device *dev = data; \
676 struct talitos_private *priv = dev_get_drvdata(dev); \
677 u32 isr, isr_lo; \
678 unsigned long flags; \
679 \
680 spin_lock_irqsave(&priv->reg_lock, flags); \
681 isr = in_be32(priv->reg + TALITOS_ISR); \
682 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
683 /* Acknowledge interrupt */ \
684 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
685 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
686 \
687 if (unlikely(isr & ch_err_mask || isr_lo)) { \
688 spin_unlock_irqrestore(&priv->reg_lock, flags); \
689 talitos_error(dev, isr & ch_err_mask, isr_lo); \
690 } \
691 else { \
692 if (likely(isr & ch_done_mask)) { \
693 /* mask further done interrupts. */ \
694 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
695 /* done_task will unmask done interrupts at exit */ \
696 tasklet_schedule(&priv->done_task[tlet]); \
697 } \
698 spin_unlock_irqrestore(&priv->reg_lock, flags); \
699 } \
700 \
701 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
702 IRQ_NONE; \
703 }
704
705 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
706 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
707 0)
708 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
709 1)
710
711 /*
712 * hwrng
713 */
714 static int talitos_rng_data_present(struct hwrng *rng, int wait)
715 {
716 struct device *dev = (struct device *)rng->priv;
717 struct talitos_private *priv = dev_get_drvdata(dev);
718 u32 ofl;
719 int i;
720
721 for (i = 0; i < 20; i++) {
722 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
723 TALITOS_RNGUSR_LO_OFL;
724 if (ofl || !wait)
725 break;
726 udelay(10);
727 }
728
729 return !!ofl;
730 }
731
732 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
733 {
734 struct device *dev = (struct device *)rng->priv;
735 struct talitos_private *priv = dev_get_drvdata(dev);
736
737 /* rng fifo requires 64-bit accesses */
738 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
739 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
740
741 return sizeof(u32);
742 }
743
744 static int talitos_rng_init(struct hwrng *rng)
745 {
746 struct device *dev = (struct device *)rng->priv;
747 struct talitos_private *priv = dev_get_drvdata(dev);
748 unsigned int timeout = TALITOS_TIMEOUT;
749
750 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
751 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
752 & TALITOS_RNGUSR_LO_RD)
753 && --timeout)
754 cpu_relax();
755 if (timeout == 0) {
756 dev_err(dev, "failed to reset rng hw\n");
757 return -ENODEV;
758 }
759
760 /* start generating */
761 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
762
763 return 0;
764 }
765
766 static int talitos_register_rng(struct device *dev)
767 {
768 struct talitos_private *priv = dev_get_drvdata(dev);
769 int err;
770
771 priv->rng.name = dev_driver_string(dev),
772 priv->rng.init = talitos_rng_init,
773 priv->rng.data_present = talitos_rng_data_present,
774 priv->rng.data_read = talitos_rng_data_read,
775 priv->rng.priv = (unsigned long)dev;
776
777 err = hwrng_register(&priv->rng);
778 if (!err)
779 priv->rng_registered = true;
780
781 return err;
782 }
783
784 static void talitos_unregister_rng(struct device *dev)
785 {
786 struct talitos_private *priv = dev_get_drvdata(dev);
787
788 if (!priv->rng_registered)
789 return;
790
791 hwrng_unregister(&priv->rng);
792 priv->rng_registered = false;
793 }
794
795 /*
796 * crypto alg
797 */
798 #define TALITOS_CRA_PRIORITY 3000
799 #define TALITOS_MAX_KEY_SIZE 96
800 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
801
802 struct talitos_ctx {
803 struct device *dev;
804 int ch;
805 __be32 desc_hdr_template;
806 u8 key[TALITOS_MAX_KEY_SIZE];
807 u8 iv[TALITOS_MAX_IV_LENGTH];
808 unsigned int keylen;
809 unsigned int enckeylen;
810 unsigned int authkeylen;
811 };
812
813 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
814 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
815
816 struct talitos_ahash_req_ctx {
817 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
818 unsigned int hw_context_size;
819 u8 buf[HASH_MAX_BLOCK_SIZE];
820 u8 bufnext[HASH_MAX_BLOCK_SIZE];
821 unsigned int swinit;
822 unsigned int first;
823 unsigned int last;
824 unsigned int to_hash_later;
825 unsigned int nbuf;
826 struct scatterlist bufsl[2];
827 struct scatterlist *psrc;
828 };
829
830 static int aead_setkey(struct crypto_aead *authenc,
831 const u8 *key, unsigned int keylen)
832 {
833 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
834 struct crypto_authenc_keys keys;
835
836 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
837 goto badkey;
838
839 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
840 goto badkey;
841
842 memcpy(ctx->key, keys.authkey, keys.authkeylen);
843 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
844
845 ctx->keylen = keys.authkeylen + keys.enckeylen;
846 ctx->enckeylen = keys.enckeylen;
847 ctx->authkeylen = keys.authkeylen;
848
849 return 0;
850
851 badkey:
852 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
853 return -EINVAL;
854 }
855
856 /*
857 * talitos_edesc - s/w-extended descriptor
858 * @src_nents: number of segments in input scatterlist
859 * @dst_nents: number of segments in output scatterlist
860 * @icv_ool: whether ICV is out-of-line
861 * @iv_dma: dma address of iv for checking continuity and link table
862 * @dma_len: length of dma mapped link_tbl space
863 * @dma_link_tbl: bus physical address of link_tbl/buf
864 * @desc: h/w descriptor
865 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
866 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
867 *
868 * if decrypting (with authcheck), or either one of src_nents or dst_nents
869 * is greater than 1, an integrity check value is concatenated to the end
870 * of link_tbl data
871 */
872 struct talitos_edesc {
873 int src_nents;
874 int dst_nents;
875 bool icv_ool;
876 dma_addr_t iv_dma;
877 int dma_len;
878 dma_addr_t dma_link_tbl;
879 struct talitos_desc desc;
880 union {
881 struct talitos_ptr link_tbl[0];
882 u8 buf[0];
883 };
884 };
885
886 static void talitos_sg_unmap(struct device *dev,
887 struct talitos_edesc *edesc,
888 struct scatterlist *src,
889 struct scatterlist *dst)
890 {
891 unsigned int src_nents = edesc->src_nents ? : 1;
892 unsigned int dst_nents = edesc->dst_nents ? : 1;
893
894 if (src != dst) {
895 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
896
897 if (dst) {
898 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
899 }
900 } else
901 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
902 }
903
904 static void ipsec_esp_unmap(struct device *dev,
905 struct talitos_edesc *edesc,
906 struct aead_request *areq)
907 {
908 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
909 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
910 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
911 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
912
913 talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
914
915 if (edesc->dma_len)
916 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
917 DMA_BIDIRECTIONAL);
918 }
919
920 /*
921 * ipsec_esp descriptor callbacks
922 */
923 static void ipsec_esp_encrypt_done(struct device *dev,
924 struct talitos_desc *desc, void *context,
925 int err)
926 {
927 struct aead_request *areq = context;
928 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
929 unsigned int authsize = crypto_aead_authsize(authenc);
930 struct talitos_edesc *edesc;
931 struct scatterlist *sg;
932 void *icvdata;
933
934 edesc = container_of(desc, struct talitos_edesc, desc);
935
936 ipsec_esp_unmap(dev, edesc, areq);
937
938 /* copy the generated ICV to dst */
939 if (edesc->icv_ool) {
940 icvdata = &edesc->link_tbl[edesc->src_nents +
941 edesc->dst_nents + 2];
942 sg = sg_last(areq->dst, edesc->dst_nents);
943 memcpy((char *)sg_virt(sg) + sg->length - authsize,
944 icvdata, authsize);
945 }
946
947 kfree(edesc);
948
949 aead_request_complete(areq, err);
950 }
951
952 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
953 struct talitos_desc *desc,
954 void *context, int err)
955 {
956 struct aead_request *req = context;
957 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
958 unsigned int authsize = crypto_aead_authsize(authenc);
959 struct talitos_edesc *edesc;
960 struct scatterlist *sg;
961 char *oicv, *icv;
962
963 edesc = container_of(desc, struct talitos_edesc, desc);
964
965 ipsec_esp_unmap(dev, edesc, req);
966
967 if (!err) {
968 /* auth check */
969 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
970 icv = (char *)sg_virt(sg) + sg->length - authsize;
971
972 if (edesc->dma_len) {
973 oicv = (char *)&edesc->link_tbl[edesc->src_nents +
974 edesc->dst_nents + 2];
975 if (edesc->icv_ool)
976 icv = oicv + authsize;
977 } else
978 oicv = (char *)&edesc->link_tbl[0];
979
980 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
981 }
982
983 kfree(edesc);
984
985 aead_request_complete(req, err);
986 }
987
988 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
989 struct talitos_desc *desc,
990 void *context, int err)
991 {
992 struct aead_request *req = context;
993 struct talitos_edesc *edesc;
994
995 edesc = container_of(desc, struct talitos_edesc, desc);
996
997 ipsec_esp_unmap(dev, edesc, req);
998
999 /* check ICV auth status */
1000 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1001 DESC_HDR_LO_ICCR1_PASS))
1002 err = -EBADMSG;
1003
1004 kfree(edesc);
1005
1006 aead_request_complete(req, err);
1007 }
1008
1009 /*
1010 * convert scatterlist to SEC h/w link table format
1011 * stop at cryptlen bytes
1012 */
1013 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1014 unsigned int offset, int cryptlen,
1015 struct talitos_ptr *link_tbl_ptr)
1016 {
1017 int n_sg = sg_count;
1018 int count = 0;
1019
1020 while (cryptlen && sg && n_sg--) {
1021 unsigned int len = sg_dma_len(sg);
1022
1023 if (offset >= len) {
1024 offset -= len;
1025 goto next;
1026 }
1027
1028 len -= offset;
1029
1030 if (len > cryptlen)
1031 len = cryptlen;
1032
1033 to_talitos_ptr(link_tbl_ptr + count,
1034 sg_dma_address(sg) + offset, 0);
1035 link_tbl_ptr[count].len = cpu_to_be16(len);
1036 link_tbl_ptr[count].j_extent = 0;
1037 count++;
1038 cryptlen -= len;
1039 offset = 0;
1040
1041 next:
1042 sg = sg_next(sg);
1043 }
1044
1045 /* tag end of link table */
1046 if (count > 0)
1047 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN;
1048
1049 return count;
1050 }
1051
1052 static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1053 int cryptlen,
1054 struct talitos_ptr *link_tbl_ptr)
1055 {
1056 return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen,
1057 link_tbl_ptr);
1058 }
1059
1060 /*
1061 * fill in and submit ipsec_esp descriptor
1062 */
1063 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1064 void (*callback)(struct device *dev,
1065 struct talitos_desc *desc,
1066 void *context, int error))
1067 {
1068 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1069 unsigned int authsize = crypto_aead_authsize(aead);
1070 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1071 struct device *dev = ctx->dev;
1072 struct talitos_desc *desc = &edesc->desc;
1073 unsigned int cryptlen = areq->cryptlen;
1074 unsigned int ivsize = crypto_aead_ivsize(aead);
1075 int tbl_off = 0;
1076 int sg_count, ret;
1077 int sg_link_tbl_len;
1078
1079 /* hmac key */
1080 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1081 DMA_TO_DEVICE);
1082
1083 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1,
1084 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1085 : DMA_TO_DEVICE);
1086
1087 /* hmac data */
1088 desc->ptr[1].len = cpu_to_be16(areq->assoclen);
1089 if (sg_count > 1 &&
1090 (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0,
1091 areq->assoclen,
1092 &edesc->link_tbl[tbl_off])) > 1) {
1093 tbl_off += ret;
1094
1095 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1096 sizeof(struct talitos_ptr), 0);
1097 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1098
1099 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1100 edesc->dma_len, DMA_BIDIRECTIONAL);
1101 } else {
1102 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0);
1103 desc->ptr[1].j_extent = 0;
1104 }
1105
1106 /* cipher iv */
1107 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1108 desc->ptr[2].len = cpu_to_be16(ivsize);
1109 desc->ptr[2].j_extent = 0;
1110
1111 /* cipher key */
1112 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1113 (char *)&ctx->key + ctx->authkeylen,
1114 DMA_TO_DEVICE);
1115
1116 /*
1117 * cipher in
1118 * map and adjust cipher len to aead request cryptlen.
1119 * extent is bytes of HMAC postpended to ciphertext,
1120 * typically 12 for ipsec
1121 */
1122 desc->ptr[4].len = cpu_to_be16(cryptlen);
1123 desc->ptr[4].j_extent = authsize;
1124
1125 sg_link_tbl_len = cryptlen;
1126 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1127 sg_link_tbl_len += authsize;
1128
1129 if (sg_count > 1 &&
1130 (ret = sg_to_link_tbl_offset(areq->src, sg_count, areq->assoclen,
1131 sg_link_tbl_len,
1132 &edesc->link_tbl[tbl_off])) > 1) {
1133 tbl_off += ret;
1134 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1135 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1136 tbl_off *
1137 sizeof(struct talitos_ptr), 0);
1138 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1139 edesc->dma_len,
1140 DMA_BIDIRECTIONAL);
1141 } else
1142 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src), 0);
1143
1144 /* cipher out */
1145 desc->ptr[5].len = cpu_to_be16(cryptlen);
1146 desc->ptr[5].j_extent = authsize;
1147
1148 if (areq->src != areq->dst)
1149 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
1150 DMA_FROM_DEVICE);
1151
1152 edesc->icv_ool = false;
1153
1154 if (sg_count > 1 &&
1155 (sg_count = sg_to_link_tbl_offset(areq->dst, sg_count,
1156 areq->assoclen, cryptlen,
1157 &edesc->link_tbl[tbl_off])) >
1158 1) {
1159 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1160
1161 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1162 tbl_off * sizeof(struct talitos_ptr), 0);
1163
1164 /* Add an entry to the link table for ICV data */
1165 tbl_ptr += sg_count - 1;
1166 tbl_ptr->j_extent = 0;
1167 tbl_ptr++;
1168 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1169 tbl_ptr->len = cpu_to_be16(authsize);
1170
1171 /* icv data follows link tables */
1172 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1173 (edesc->src_nents + edesc->dst_nents +
1174 2) * sizeof(struct talitos_ptr) +
1175 authsize, 0);
1176 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1177 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1178 edesc->dma_len, DMA_BIDIRECTIONAL);
1179
1180 edesc->icv_ool = true;
1181 } else
1182 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0);
1183
1184 /* iv out */
1185 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1186 DMA_FROM_DEVICE);
1187
1188 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1189 if (ret != -EINPROGRESS) {
1190 ipsec_esp_unmap(dev, edesc, areq);
1191 kfree(edesc);
1192 }
1193 return ret;
1194 }
1195
1196 /*
1197 * allocate and map the extended descriptor
1198 */
1199 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1200 struct scatterlist *src,
1201 struct scatterlist *dst,
1202 u8 *iv,
1203 unsigned int assoclen,
1204 unsigned int cryptlen,
1205 unsigned int authsize,
1206 unsigned int ivsize,
1207 int icv_stashing,
1208 u32 cryptoflags,
1209 bool encrypt)
1210 {
1211 struct talitos_edesc *edesc;
1212 int src_nents, dst_nents, alloc_len, dma_len;
1213 dma_addr_t iv_dma = 0;
1214 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1215 GFP_ATOMIC;
1216 struct talitos_private *priv = dev_get_drvdata(dev);
1217 bool is_sec1 = has_ftr_sec1(priv);
1218 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1219
1220 if (cryptlen + authsize > max_len) {
1221 dev_err(dev, "length exceeds h/w max limit\n");
1222 return ERR_PTR(-EINVAL);
1223 }
1224
1225 if (ivsize)
1226 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1227
1228 if (!dst || dst == src) {
1229 src_nents = sg_nents_for_len(src,
1230 assoclen + cryptlen + authsize);
1231 src_nents = (src_nents == 1) ? 0 : src_nents;
1232 dst_nents = dst ? src_nents : 0;
1233 } else { /* dst && dst != src*/
1234 src_nents = sg_nents_for_len(src, assoclen + cryptlen +
1235 (encrypt ? 0 : authsize));
1236 src_nents = (src_nents == 1) ? 0 : src_nents;
1237 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen +
1238 (encrypt ? authsize : 0));
1239 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1240 }
1241
1242 /*
1243 * allocate space for base edesc plus the link tables,
1244 * allowing for two separate entries for AD and generated ICV (+ 2),
1245 * and space for two sets of ICVs (stashed and generated)
1246 */
1247 alloc_len = sizeof(struct talitos_edesc);
1248 if (src_nents || dst_nents) {
1249 if (is_sec1)
1250 dma_len = (src_nents ? cryptlen : 0) +
1251 (dst_nents ? cryptlen : 0);
1252 else
1253 dma_len = (src_nents + dst_nents + 2) *
1254 sizeof(struct talitos_ptr) + authsize * 2;
1255 alloc_len += dma_len;
1256 } else {
1257 dma_len = 0;
1258 alloc_len += icv_stashing ? authsize : 0;
1259 }
1260
1261 edesc = kmalloc(alloc_len, GFP_DMA | flags);
1262 if (!edesc) {
1263 if (iv_dma)
1264 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1265
1266 dev_err(dev, "could not allocate edescriptor\n");
1267 return ERR_PTR(-ENOMEM);
1268 }
1269
1270 edesc->src_nents = src_nents;
1271 edesc->dst_nents = dst_nents;
1272 edesc->iv_dma = iv_dma;
1273 edesc->dma_len = dma_len;
1274 if (dma_len)
1275 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1276 edesc->dma_len,
1277 DMA_BIDIRECTIONAL);
1278
1279 return edesc;
1280 }
1281
1282 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1283 int icv_stashing, bool encrypt)
1284 {
1285 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1286 unsigned int authsize = crypto_aead_authsize(authenc);
1287 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1288 unsigned int ivsize = crypto_aead_ivsize(authenc);
1289
1290 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1291 iv, areq->assoclen, areq->cryptlen,
1292 authsize, ivsize, icv_stashing,
1293 areq->base.flags, encrypt);
1294 }
1295
1296 static int aead_encrypt(struct aead_request *req)
1297 {
1298 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1299 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1300 struct talitos_edesc *edesc;
1301
1302 /* allocate extended descriptor */
1303 edesc = aead_edesc_alloc(req, req->iv, 0, true);
1304 if (IS_ERR(edesc))
1305 return PTR_ERR(edesc);
1306
1307 /* set encrypt */
1308 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1309
1310 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1311 }
1312
1313 static int aead_decrypt(struct aead_request *req)
1314 {
1315 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1316 unsigned int authsize = crypto_aead_authsize(authenc);
1317 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1318 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1319 struct talitos_edesc *edesc;
1320 struct scatterlist *sg;
1321 void *icvdata;
1322
1323 req->cryptlen -= authsize;
1324
1325 /* allocate extended descriptor */
1326 edesc = aead_edesc_alloc(req, req->iv, 1, false);
1327 if (IS_ERR(edesc))
1328 return PTR_ERR(edesc);
1329
1330 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1331 ((!edesc->src_nents && !edesc->dst_nents) ||
1332 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1333
1334 /* decrypt and check the ICV */
1335 edesc->desc.hdr = ctx->desc_hdr_template |
1336 DESC_HDR_DIR_INBOUND |
1337 DESC_HDR_MODE1_MDEU_CICV;
1338
1339 /* reset integrity check result bits */
1340 edesc->desc.hdr_lo = 0;
1341
1342 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1343 }
1344
1345 /* Have to check the ICV with software */
1346 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1347
1348 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1349 if (edesc->dma_len)
1350 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1351 edesc->dst_nents + 2];
1352 else
1353 icvdata = &edesc->link_tbl[0];
1354
1355 sg = sg_last(req->src, edesc->src_nents ? : 1);
1356
1357 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1358
1359 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1360 }
1361
1362 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1363 const u8 *key, unsigned int keylen)
1364 {
1365 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1366
1367 memcpy(&ctx->key, key, keylen);
1368 ctx->keylen = keylen;
1369
1370 return 0;
1371 }
1372
1373 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1374 struct scatterlist *dst, unsigned int len,
1375 struct talitos_edesc *edesc)
1376 {
1377 struct talitos_private *priv = dev_get_drvdata(dev);
1378 bool is_sec1 = has_ftr_sec1(priv);
1379
1380 if (is_sec1) {
1381 if (!edesc->src_nents) {
1382 dma_unmap_sg(dev, src, 1,
1383 dst != src ? DMA_TO_DEVICE
1384 : DMA_BIDIRECTIONAL);
1385 }
1386 if (dst && edesc->dst_nents) {
1387 dma_sync_single_for_device(dev,
1388 edesc->dma_link_tbl + len,
1389 len, DMA_FROM_DEVICE);
1390 sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1391 edesc->buf + len, len);
1392 } else if (dst && dst != src) {
1393 dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1394 }
1395 } else {
1396 talitos_sg_unmap(dev, edesc, src, dst);
1397 }
1398 }
1399
1400 static void common_nonsnoop_unmap(struct device *dev,
1401 struct talitos_edesc *edesc,
1402 struct ablkcipher_request *areq)
1403 {
1404 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1405
1406 unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1407 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1408 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1409
1410 if (edesc->dma_len)
1411 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1412 DMA_BIDIRECTIONAL);
1413 }
1414
1415 static void ablkcipher_done(struct device *dev,
1416 struct talitos_desc *desc, void *context,
1417 int err)
1418 {
1419 struct ablkcipher_request *areq = context;
1420 struct talitos_edesc *edesc;
1421
1422 edesc = container_of(desc, struct talitos_edesc, desc);
1423
1424 common_nonsnoop_unmap(dev, edesc, areq);
1425
1426 kfree(edesc);
1427
1428 areq->base.complete(&areq->base, err);
1429 }
1430
1431 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1432 unsigned int len, struct talitos_edesc *edesc,
1433 enum dma_data_direction dir, struct talitos_ptr *ptr)
1434 {
1435 int sg_count;
1436 struct talitos_private *priv = dev_get_drvdata(dev);
1437 bool is_sec1 = has_ftr_sec1(priv);
1438
1439 to_talitos_ptr_len(ptr, len, is_sec1);
1440
1441 if (is_sec1) {
1442 sg_count = edesc->src_nents ? : 1;
1443
1444 if (sg_count == 1) {
1445 dma_map_sg(dev, src, 1, dir);
1446 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1447 } else {
1448 sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1449 to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1450 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1451 len, DMA_TO_DEVICE);
1452 }
1453 } else {
1454 to_talitos_ptr_extent_clear(ptr, is_sec1);
1455
1456 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir);
1457
1458 if (sg_count == 1) {
1459 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1460 } else {
1461 sg_count = sg_to_link_tbl(src, sg_count, len,
1462 &edesc->link_tbl[0]);
1463 if (sg_count > 1) {
1464 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1465 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1466 dma_sync_single_for_device(dev,
1467 edesc->dma_link_tbl,
1468 edesc->dma_len,
1469 DMA_BIDIRECTIONAL);
1470 } else {
1471 /* Only one segment now, so no link tbl needed*/
1472 to_talitos_ptr(ptr, sg_dma_address(src),
1473 is_sec1);
1474 }
1475 }
1476 }
1477 return sg_count;
1478 }
1479
1480 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1481 unsigned int len, struct talitos_edesc *edesc,
1482 enum dma_data_direction dir,
1483 struct talitos_ptr *ptr, int sg_count)
1484 {
1485 struct talitos_private *priv = dev_get_drvdata(dev);
1486 bool is_sec1 = has_ftr_sec1(priv);
1487
1488 if (dir != DMA_NONE)
1489 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir);
1490
1491 to_talitos_ptr_len(ptr, len, is_sec1);
1492
1493 if (is_sec1) {
1494 if (sg_count == 1) {
1495 if (dir != DMA_NONE)
1496 dma_map_sg(dev, dst, 1, dir);
1497 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1498 } else {
1499 to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1500 dma_sync_single_for_device(dev,
1501 edesc->dma_link_tbl + len,
1502 len, DMA_FROM_DEVICE);
1503 }
1504 } else {
1505 to_talitos_ptr_extent_clear(ptr, is_sec1);
1506
1507 if (sg_count == 1) {
1508 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1509 } else {
1510 struct talitos_ptr *link_tbl_ptr =
1511 &edesc->link_tbl[edesc->src_nents + 1];
1512
1513 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1514 (edesc->src_nents + 1) *
1515 sizeof(struct talitos_ptr), 0);
1516 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1517 sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1518 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1519 edesc->dma_len,
1520 DMA_BIDIRECTIONAL);
1521 }
1522 }
1523 }
1524
1525 static int common_nonsnoop(struct talitos_edesc *edesc,
1526 struct ablkcipher_request *areq,
1527 void (*callback) (struct device *dev,
1528 struct talitos_desc *desc,
1529 void *context, int error))
1530 {
1531 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1532 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1533 struct device *dev = ctx->dev;
1534 struct talitos_desc *desc = &edesc->desc;
1535 unsigned int cryptlen = areq->nbytes;
1536 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1537 int sg_count, ret;
1538 struct talitos_private *priv = dev_get_drvdata(dev);
1539 bool is_sec1 = has_ftr_sec1(priv);
1540
1541 /* first DWORD empty */
1542 desc->ptr[0] = zero_entry;
1543
1544 /* cipher iv */
1545 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1546 to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1547 to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1548
1549 /* cipher key */
1550 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1551 (char *)&ctx->key, DMA_TO_DEVICE);
1552
1553 /*
1554 * cipher in
1555 */
1556 sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1557 (areq->src == areq->dst) ?
1558 DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1559 &desc->ptr[3]);
1560
1561 /* cipher out */
1562 map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1563 (areq->src == areq->dst) ? DMA_NONE
1564 : DMA_FROM_DEVICE,
1565 &desc->ptr[4], sg_count);
1566
1567 /* iv out */
1568 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1569 DMA_FROM_DEVICE);
1570
1571 /* last DWORD empty */
1572 desc->ptr[6] = zero_entry;
1573
1574 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1575 if (ret != -EINPROGRESS) {
1576 common_nonsnoop_unmap(dev, edesc, areq);
1577 kfree(edesc);
1578 }
1579 return ret;
1580 }
1581
1582 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1583 areq, bool encrypt)
1584 {
1585 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1586 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1587 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1588
1589 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1590 areq->info, 0, areq->nbytes, 0, ivsize, 0,
1591 areq->base.flags, encrypt);
1592 }
1593
1594 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1595 {
1596 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1597 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1598 struct talitos_edesc *edesc;
1599
1600 /* allocate extended descriptor */
1601 edesc = ablkcipher_edesc_alloc(areq, true);
1602 if (IS_ERR(edesc))
1603 return PTR_ERR(edesc);
1604
1605 /* set encrypt */
1606 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1607
1608 return common_nonsnoop(edesc, areq, ablkcipher_done);
1609 }
1610
1611 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1612 {
1613 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1614 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1615 struct talitos_edesc *edesc;
1616
1617 /* allocate extended descriptor */
1618 edesc = ablkcipher_edesc_alloc(areq, false);
1619 if (IS_ERR(edesc))
1620 return PTR_ERR(edesc);
1621
1622 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1623
1624 return common_nonsnoop(edesc, areq, ablkcipher_done);
1625 }
1626
1627 static void common_nonsnoop_hash_unmap(struct device *dev,
1628 struct talitos_edesc *edesc,
1629 struct ahash_request *areq)
1630 {
1631 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1632 struct talitos_private *priv = dev_get_drvdata(dev);
1633 bool is_sec1 = has_ftr_sec1(priv);
1634
1635 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1636
1637 unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1638
1639 /* When using hashctx-in, must unmap it. */
1640 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1641 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1642 DMA_TO_DEVICE);
1643
1644 if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1645 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1646 DMA_TO_DEVICE);
1647
1648 if (edesc->dma_len)
1649 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1650 DMA_BIDIRECTIONAL);
1651
1652 }
1653
1654 static void ahash_done(struct device *dev,
1655 struct talitos_desc *desc, void *context,
1656 int err)
1657 {
1658 struct ahash_request *areq = context;
1659 struct talitos_edesc *edesc =
1660 container_of(desc, struct talitos_edesc, desc);
1661 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1662
1663 if (!req_ctx->last && req_ctx->to_hash_later) {
1664 /* Position any partial block for next update/final/finup */
1665 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1666 req_ctx->nbuf = req_ctx->to_hash_later;
1667 }
1668 common_nonsnoop_hash_unmap(dev, edesc, areq);
1669
1670 kfree(edesc);
1671
1672 areq->base.complete(&areq->base, err);
1673 }
1674
1675 /*
1676 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1677 * ourself and submit a padded block
1678 */
1679 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1680 struct talitos_edesc *edesc,
1681 struct talitos_ptr *ptr)
1682 {
1683 static u8 padded_hash[64] = {
1684 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1685 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1686 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1687 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1688 };
1689
1690 pr_err_once("Bug in SEC1, padding ourself\n");
1691 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1692 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1693 (char *)padded_hash, DMA_TO_DEVICE);
1694 }
1695
1696 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1697 struct ahash_request *areq, unsigned int length,
1698 void (*callback) (struct device *dev,
1699 struct talitos_desc *desc,
1700 void *context, int error))
1701 {
1702 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1703 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1704 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1705 struct device *dev = ctx->dev;
1706 struct talitos_desc *desc = &edesc->desc;
1707 int ret;
1708 struct talitos_private *priv = dev_get_drvdata(dev);
1709 bool is_sec1 = has_ftr_sec1(priv);
1710
1711 /* first DWORD empty */
1712 desc->ptr[0] = zero_entry;
1713
1714 /* hash context in */
1715 if (!req_ctx->first || req_ctx->swinit) {
1716 map_single_talitos_ptr(dev, &desc->ptr[1],
1717 req_ctx->hw_context_size,
1718 (char *)req_ctx->hw_context,
1719 DMA_TO_DEVICE);
1720 req_ctx->swinit = 0;
1721 } else {
1722 desc->ptr[1] = zero_entry;
1723 /* Indicate next op is not the first. */
1724 req_ctx->first = 0;
1725 }
1726
1727 /* HMAC key */
1728 if (ctx->keylen)
1729 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1730 (char *)&ctx->key, DMA_TO_DEVICE);
1731 else
1732 desc->ptr[2] = zero_entry;
1733
1734 /*
1735 * data in
1736 */
1737 map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1738 DMA_TO_DEVICE, &desc->ptr[3]);
1739
1740 /* fifth DWORD empty */
1741 desc->ptr[4] = zero_entry;
1742
1743 /* hash/HMAC out -or- hash context out */
1744 if (req_ctx->last)
1745 map_single_talitos_ptr(dev, &desc->ptr[5],
1746 crypto_ahash_digestsize(tfm),
1747 areq->result, DMA_FROM_DEVICE);
1748 else
1749 map_single_talitos_ptr(dev, &desc->ptr[5],
1750 req_ctx->hw_context_size,
1751 req_ctx->hw_context, DMA_FROM_DEVICE);
1752
1753 /* last DWORD empty */
1754 desc->ptr[6] = zero_entry;
1755
1756 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1757 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1758
1759 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1760 if (ret != -EINPROGRESS) {
1761 common_nonsnoop_hash_unmap(dev, edesc, areq);
1762 kfree(edesc);
1763 }
1764 return ret;
1765 }
1766
1767 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1768 unsigned int nbytes)
1769 {
1770 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1771 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1772 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1773
1774 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1775 nbytes, 0, 0, 0, areq->base.flags, false);
1776 }
1777
1778 static int ahash_init(struct ahash_request *areq)
1779 {
1780 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1781 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1782
1783 /* Initialize the context */
1784 req_ctx->nbuf = 0;
1785 req_ctx->first = 1; /* first indicates h/w must init its context */
1786 req_ctx->swinit = 0; /* assume h/w init of context */
1787 req_ctx->hw_context_size =
1788 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1789 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1790 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1791
1792 return 0;
1793 }
1794
1795 /*
1796 * on h/w without explicit sha224 support, we initialize h/w context
1797 * manually with sha224 constants, and tell it to run sha256.
1798 */
1799 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1800 {
1801 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1802
1803 ahash_init(areq);
1804 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1805
1806 req_ctx->hw_context[0] = SHA224_H0;
1807 req_ctx->hw_context[1] = SHA224_H1;
1808 req_ctx->hw_context[2] = SHA224_H2;
1809 req_ctx->hw_context[3] = SHA224_H3;
1810 req_ctx->hw_context[4] = SHA224_H4;
1811 req_ctx->hw_context[5] = SHA224_H5;
1812 req_ctx->hw_context[6] = SHA224_H6;
1813 req_ctx->hw_context[7] = SHA224_H7;
1814
1815 /* init 64-bit count */
1816 req_ctx->hw_context[8] = 0;
1817 req_ctx->hw_context[9] = 0;
1818
1819 return 0;
1820 }
1821
1822 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1823 {
1824 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1825 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1826 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1827 struct talitos_edesc *edesc;
1828 unsigned int blocksize =
1829 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1830 unsigned int nbytes_to_hash;
1831 unsigned int to_hash_later;
1832 unsigned int nsg;
1833
1834 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1835 /* Buffer up to one whole block */
1836 sg_copy_to_buffer(areq->src,
1837 sg_nents_for_len(areq->src, nbytes),
1838 req_ctx->buf + req_ctx->nbuf, nbytes);
1839 req_ctx->nbuf += nbytes;
1840 return 0;
1841 }
1842
1843 /* At least (blocksize + 1) bytes are available to hash */
1844 nbytes_to_hash = nbytes + req_ctx->nbuf;
1845 to_hash_later = nbytes_to_hash & (blocksize - 1);
1846
1847 if (req_ctx->last)
1848 to_hash_later = 0;
1849 else if (to_hash_later)
1850 /* There is a partial block. Hash the full block(s) now */
1851 nbytes_to_hash -= to_hash_later;
1852 else {
1853 /* Keep one block buffered */
1854 nbytes_to_hash -= blocksize;
1855 to_hash_later = blocksize;
1856 }
1857
1858 /* Chain in any previously buffered data */
1859 if (req_ctx->nbuf) {
1860 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1861 sg_init_table(req_ctx->bufsl, nsg);
1862 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1863 if (nsg > 1)
1864 sg_chain(req_ctx->bufsl, 2, areq->src);
1865 req_ctx->psrc = req_ctx->bufsl;
1866 } else
1867 req_ctx->psrc = areq->src;
1868
1869 if (to_hash_later) {
1870 int nents = sg_nents_for_len(areq->src, nbytes);
1871 sg_pcopy_to_buffer(areq->src, nents,
1872 req_ctx->bufnext,
1873 to_hash_later,
1874 nbytes - to_hash_later);
1875 }
1876 req_ctx->to_hash_later = to_hash_later;
1877
1878 /* Allocate extended descriptor */
1879 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1880 if (IS_ERR(edesc))
1881 return PTR_ERR(edesc);
1882
1883 edesc->desc.hdr = ctx->desc_hdr_template;
1884
1885 /* On last one, request SEC to pad; otherwise continue */
1886 if (req_ctx->last)
1887 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1888 else
1889 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1890
1891 /* request SEC to INIT hash. */
1892 if (req_ctx->first && !req_ctx->swinit)
1893 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1894
1895 /* When the tfm context has a keylen, it's an HMAC.
1896 * A first or last (ie. not middle) descriptor must request HMAC.
1897 */
1898 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1899 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1900
1901 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1902 ahash_done);
1903 }
1904
1905 static int ahash_update(struct ahash_request *areq)
1906 {
1907 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1908
1909 req_ctx->last = 0;
1910
1911 return ahash_process_req(areq, areq->nbytes);
1912 }
1913
1914 static int ahash_final(struct ahash_request *areq)
1915 {
1916 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1917
1918 req_ctx->last = 1;
1919
1920 return ahash_process_req(areq, 0);
1921 }
1922
1923 static int ahash_finup(struct ahash_request *areq)
1924 {
1925 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1926
1927 req_ctx->last = 1;
1928
1929 return ahash_process_req(areq, areq->nbytes);
1930 }
1931
1932 static int ahash_digest(struct ahash_request *areq)
1933 {
1934 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1935 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1936
1937 ahash->init(areq);
1938 req_ctx->last = 1;
1939
1940 return ahash_process_req(areq, areq->nbytes);
1941 }
1942
1943 struct keyhash_result {
1944 struct completion completion;
1945 int err;
1946 };
1947
1948 static void keyhash_complete(struct crypto_async_request *req, int err)
1949 {
1950 struct keyhash_result *res = req->data;
1951
1952 if (err == -EINPROGRESS)
1953 return;
1954
1955 res->err = err;
1956 complete(&res->completion);
1957 }
1958
1959 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
1960 u8 *hash)
1961 {
1962 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1963
1964 struct scatterlist sg[1];
1965 struct ahash_request *req;
1966 struct keyhash_result hresult;
1967 int ret;
1968
1969 init_completion(&hresult.completion);
1970
1971 req = ahash_request_alloc(tfm, GFP_KERNEL);
1972 if (!req)
1973 return -ENOMEM;
1974
1975 /* Keep tfm keylen == 0 during hash of the long key */
1976 ctx->keylen = 0;
1977 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1978 keyhash_complete, &hresult);
1979
1980 sg_init_one(&sg[0], key, keylen);
1981
1982 ahash_request_set_crypt(req, sg, hash, keylen);
1983 ret = crypto_ahash_digest(req);
1984 switch (ret) {
1985 case 0:
1986 break;
1987 case -EINPROGRESS:
1988 case -EBUSY:
1989 ret = wait_for_completion_interruptible(
1990 &hresult.completion);
1991 if (!ret)
1992 ret = hresult.err;
1993 break;
1994 default:
1995 break;
1996 }
1997 ahash_request_free(req);
1998
1999 return ret;
2000 }
2001
2002 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2003 unsigned int keylen)
2004 {
2005 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2006 unsigned int blocksize =
2007 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2008 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2009 unsigned int keysize = keylen;
2010 u8 hash[SHA512_DIGEST_SIZE];
2011 int ret;
2012
2013 if (keylen <= blocksize)
2014 memcpy(ctx->key, key, keysize);
2015 else {
2016 /* Must get the hash of the long key */
2017 ret = keyhash(tfm, key, keylen, hash);
2018
2019 if (ret) {
2020 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2021 return -EINVAL;
2022 }
2023
2024 keysize = digestsize;
2025 memcpy(ctx->key, hash, digestsize);
2026 }
2027
2028 ctx->keylen = keysize;
2029
2030 return 0;
2031 }
2032
2033
2034 struct talitos_alg_template {
2035 u32 type;
2036 union {
2037 struct crypto_alg crypto;
2038 struct ahash_alg hash;
2039 struct aead_alg aead;
2040 } alg;
2041 __be32 desc_hdr_template;
2042 };
2043
2044 static struct talitos_alg_template driver_algs[] = {
2045 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2046 { .type = CRYPTO_ALG_TYPE_AEAD,
2047 .alg.aead = {
2048 .base = {
2049 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2050 .cra_driver_name = "authenc-hmac-sha1-"
2051 "cbc-aes-talitos",
2052 .cra_blocksize = AES_BLOCK_SIZE,
2053 .cra_flags = CRYPTO_ALG_ASYNC,
2054 },
2055 .ivsize = AES_BLOCK_SIZE,
2056 .maxauthsize = SHA1_DIGEST_SIZE,
2057 },
2058 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2059 DESC_HDR_SEL0_AESU |
2060 DESC_HDR_MODE0_AESU_CBC |
2061 DESC_HDR_SEL1_MDEUA |
2062 DESC_HDR_MODE1_MDEU_INIT |
2063 DESC_HDR_MODE1_MDEU_PAD |
2064 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2065 },
2066 { .type = CRYPTO_ALG_TYPE_AEAD,
2067 .alg.aead = {
2068 .base = {
2069 .cra_name = "authenc(hmac(sha1),"
2070 "cbc(des3_ede))",
2071 .cra_driver_name = "authenc-hmac-sha1-"
2072 "cbc-3des-talitos",
2073 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2074 .cra_flags = CRYPTO_ALG_ASYNC,
2075 },
2076 .ivsize = DES3_EDE_BLOCK_SIZE,
2077 .maxauthsize = SHA1_DIGEST_SIZE,
2078 },
2079 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2080 DESC_HDR_SEL0_DEU |
2081 DESC_HDR_MODE0_DEU_CBC |
2082 DESC_HDR_MODE0_DEU_3DES |
2083 DESC_HDR_SEL1_MDEUA |
2084 DESC_HDR_MODE1_MDEU_INIT |
2085 DESC_HDR_MODE1_MDEU_PAD |
2086 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2087 },
2088 { .type = CRYPTO_ALG_TYPE_AEAD,
2089 .alg.aead = {
2090 .base = {
2091 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2092 .cra_driver_name = "authenc-hmac-sha224-"
2093 "cbc-aes-talitos",
2094 .cra_blocksize = AES_BLOCK_SIZE,
2095 .cra_flags = CRYPTO_ALG_ASYNC,
2096 },
2097 .ivsize = AES_BLOCK_SIZE,
2098 .maxauthsize = SHA224_DIGEST_SIZE,
2099 },
2100 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2101 DESC_HDR_SEL0_AESU |
2102 DESC_HDR_MODE0_AESU_CBC |
2103 DESC_HDR_SEL1_MDEUA |
2104 DESC_HDR_MODE1_MDEU_INIT |
2105 DESC_HDR_MODE1_MDEU_PAD |
2106 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2107 },
2108 { .type = CRYPTO_ALG_TYPE_AEAD,
2109 .alg.aead = {
2110 .base = {
2111 .cra_name = "authenc(hmac(sha224),"
2112 "cbc(des3_ede))",
2113 .cra_driver_name = "authenc-hmac-sha224-"
2114 "cbc-3des-talitos",
2115 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2116 .cra_flags = CRYPTO_ALG_ASYNC,
2117 },
2118 .ivsize = DES3_EDE_BLOCK_SIZE,
2119 .maxauthsize = SHA224_DIGEST_SIZE,
2120 },
2121 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2122 DESC_HDR_SEL0_DEU |
2123 DESC_HDR_MODE0_DEU_CBC |
2124 DESC_HDR_MODE0_DEU_3DES |
2125 DESC_HDR_SEL1_MDEUA |
2126 DESC_HDR_MODE1_MDEU_INIT |
2127 DESC_HDR_MODE1_MDEU_PAD |
2128 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2129 },
2130 { .type = CRYPTO_ALG_TYPE_AEAD,
2131 .alg.aead = {
2132 .base = {
2133 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2134 .cra_driver_name = "authenc-hmac-sha256-"
2135 "cbc-aes-talitos",
2136 .cra_blocksize = AES_BLOCK_SIZE,
2137 .cra_flags = CRYPTO_ALG_ASYNC,
2138 },
2139 .ivsize = AES_BLOCK_SIZE,
2140 .maxauthsize = SHA256_DIGEST_SIZE,
2141 },
2142 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2143 DESC_HDR_SEL0_AESU |
2144 DESC_HDR_MODE0_AESU_CBC |
2145 DESC_HDR_SEL1_MDEUA |
2146 DESC_HDR_MODE1_MDEU_INIT |
2147 DESC_HDR_MODE1_MDEU_PAD |
2148 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2149 },
2150 { .type = CRYPTO_ALG_TYPE_AEAD,
2151 .alg.aead = {
2152 .base = {
2153 .cra_name = "authenc(hmac(sha256),"
2154 "cbc(des3_ede))",
2155 .cra_driver_name = "authenc-hmac-sha256-"
2156 "cbc-3des-talitos",
2157 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2158 .cra_flags = CRYPTO_ALG_ASYNC,
2159 },
2160 .ivsize = DES3_EDE_BLOCK_SIZE,
2161 .maxauthsize = SHA256_DIGEST_SIZE,
2162 },
2163 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2164 DESC_HDR_SEL0_DEU |
2165 DESC_HDR_MODE0_DEU_CBC |
2166 DESC_HDR_MODE0_DEU_3DES |
2167 DESC_HDR_SEL1_MDEUA |
2168 DESC_HDR_MODE1_MDEU_INIT |
2169 DESC_HDR_MODE1_MDEU_PAD |
2170 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2171 },
2172 { .type = CRYPTO_ALG_TYPE_AEAD,
2173 .alg.aead = {
2174 .base = {
2175 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2176 .cra_driver_name = "authenc-hmac-sha384-"
2177 "cbc-aes-talitos",
2178 .cra_blocksize = AES_BLOCK_SIZE,
2179 .cra_flags = CRYPTO_ALG_ASYNC,
2180 },
2181 .ivsize = AES_BLOCK_SIZE,
2182 .maxauthsize = SHA384_DIGEST_SIZE,
2183 },
2184 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2185 DESC_HDR_SEL0_AESU |
2186 DESC_HDR_MODE0_AESU_CBC |
2187 DESC_HDR_SEL1_MDEUB |
2188 DESC_HDR_MODE1_MDEU_INIT |
2189 DESC_HDR_MODE1_MDEU_PAD |
2190 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2191 },
2192 { .type = CRYPTO_ALG_TYPE_AEAD,
2193 .alg.aead = {
2194 .base = {
2195 .cra_name = "authenc(hmac(sha384),"
2196 "cbc(des3_ede))",
2197 .cra_driver_name = "authenc-hmac-sha384-"
2198 "cbc-3des-talitos",
2199 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2200 .cra_flags = CRYPTO_ALG_ASYNC,
2201 },
2202 .ivsize = DES3_EDE_BLOCK_SIZE,
2203 .maxauthsize = SHA384_DIGEST_SIZE,
2204 },
2205 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2206 DESC_HDR_SEL0_DEU |
2207 DESC_HDR_MODE0_DEU_CBC |
2208 DESC_HDR_MODE0_DEU_3DES |
2209 DESC_HDR_SEL1_MDEUB |
2210 DESC_HDR_MODE1_MDEU_INIT |
2211 DESC_HDR_MODE1_MDEU_PAD |
2212 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2213 },
2214 { .type = CRYPTO_ALG_TYPE_AEAD,
2215 .alg.aead = {
2216 .base = {
2217 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2218 .cra_driver_name = "authenc-hmac-sha512-"
2219 "cbc-aes-talitos",
2220 .cra_blocksize = AES_BLOCK_SIZE,
2221 .cra_flags = CRYPTO_ALG_ASYNC,
2222 },
2223 .ivsize = AES_BLOCK_SIZE,
2224 .maxauthsize = SHA512_DIGEST_SIZE,
2225 },
2226 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2227 DESC_HDR_SEL0_AESU |
2228 DESC_HDR_MODE0_AESU_CBC |
2229 DESC_HDR_SEL1_MDEUB |
2230 DESC_HDR_MODE1_MDEU_INIT |
2231 DESC_HDR_MODE1_MDEU_PAD |
2232 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2233 },
2234 { .type = CRYPTO_ALG_TYPE_AEAD,
2235 .alg.aead = {
2236 .base = {
2237 .cra_name = "authenc(hmac(sha512),"
2238 "cbc(des3_ede))",
2239 .cra_driver_name = "authenc-hmac-sha512-"
2240 "cbc-3des-talitos",
2241 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2242 .cra_flags = CRYPTO_ALG_ASYNC,
2243 },
2244 .ivsize = DES3_EDE_BLOCK_SIZE,
2245 .maxauthsize = SHA512_DIGEST_SIZE,
2246 },
2247 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2248 DESC_HDR_SEL0_DEU |
2249 DESC_HDR_MODE0_DEU_CBC |
2250 DESC_HDR_MODE0_DEU_3DES |
2251 DESC_HDR_SEL1_MDEUB |
2252 DESC_HDR_MODE1_MDEU_INIT |
2253 DESC_HDR_MODE1_MDEU_PAD |
2254 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2255 },
2256 { .type = CRYPTO_ALG_TYPE_AEAD,
2257 .alg.aead = {
2258 .base = {
2259 .cra_name = "authenc(hmac(md5),cbc(aes))",
2260 .cra_driver_name = "authenc-hmac-md5-"
2261 "cbc-aes-talitos",
2262 .cra_blocksize = AES_BLOCK_SIZE,
2263 .cra_flags = CRYPTO_ALG_ASYNC,
2264 },
2265 .ivsize = AES_BLOCK_SIZE,
2266 .maxauthsize = MD5_DIGEST_SIZE,
2267 },
2268 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2269 DESC_HDR_SEL0_AESU |
2270 DESC_HDR_MODE0_AESU_CBC |
2271 DESC_HDR_SEL1_MDEUA |
2272 DESC_HDR_MODE1_MDEU_INIT |
2273 DESC_HDR_MODE1_MDEU_PAD |
2274 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2275 },
2276 { .type = CRYPTO_ALG_TYPE_AEAD,
2277 .alg.aead = {
2278 .base = {
2279 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2280 .cra_driver_name = "authenc-hmac-md5-"
2281 "cbc-3des-talitos",
2282 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283 .cra_flags = CRYPTO_ALG_ASYNC,
2284 },
2285 .ivsize = DES3_EDE_BLOCK_SIZE,
2286 .maxauthsize = MD5_DIGEST_SIZE,
2287 },
2288 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2289 DESC_HDR_SEL0_DEU |
2290 DESC_HDR_MODE0_DEU_CBC |
2291 DESC_HDR_MODE0_DEU_3DES |
2292 DESC_HDR_SEL1_MDEUA |
2293 DESC_HDR_MODE1_MDEU_INIT |
2294 DESC_HDR_MODE1_MDEU_PAD |
2295 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2296 },
2297 /* ABLKCIPHER algorithms. */
2298 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2299 .alg.crypto = {
2300 .cra_name = "cbc(aes)",
2301 .cra_driver_name = "cbc-aes-talitos",
2302 .cra_blocksize = AES_BLOCK_SIZE,
2303 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2304 CRYPTO_ALG_ASYNC,
2305 .cra_ablkcipher = {
2306 .min_keysize = AES_MIN_KEY_SIZE,
2307 .max_keysize = AES_MAX_KEY_SIZE,
2308 .ivsize = AES_BLOCK_SIZE,
2309 }
2310 },
2311 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2312 DESC_HDR_SEL0_AESU |
2313 DESC_HDR_MODE0_AESU_CBC,
2314 },
2315 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2316 .alg.crypto = {
2317 .cra_name = "cbc(des3_ede)",
2318 .cra_driver_name = "cbc-3des-talitos",
2319 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2320 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2321 CRYPTO_ALG_ASYNC,
2322 .cra_ablkcipher = {
2323 .min_keysize = DES3_EDE_KEY_SIZE,
2324 .max_keysize = DES3_EDE_KEY_SIZE,
2325 .ivsize = DES3_EDE_BLOCK_SIZE,
2326 }
2327 },
2328 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2329 DESC_HDR_SEL0_DEU |
2330 DESC_HDR_MODE0_DEU_CBC |
2331 DESC_HDR_MODE0_DEU_3DES,
2332 },
2333 /* AHASH algorithms. */
2334 { .type = CRYPTO_ALG_TYPE_AHASH,
2335 .alg.hash = {
2336 .halg.digestsize = MD5_DIGEST_SIZE,
2337 .halg.base = {
2338 .cra_name = "md5",
2339 .cra_driver_name = "md5-talitos",
2340 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2341 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2342 CRYPTO_ALG_ASYNC,
2343 }
2344 },
2345 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2346 DESC_HDR_SEL0_MDEUA |
2347 DESC_HDR_MODE0_MDEU_MD5,
2348 },
2349 { .type = CRYPTO_ALG_TYPE_AHASH,
2350 .alg.hash = {
2351 .halg.digestsize = SHA1_DIGEST_SIZE,
2352 .halg.base = {
2353 .cra_name = "sha1",
2354 .cra_driver_name = "sha1-talitos",
2355 .cra_blocksize = SHA1_BLOCK_SIZE,
2356 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2357 CRYPTO_ALG_ASYNC,
2358 }
2359 },
2360 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2361 DESC_HDR_SEL0_MDEUA |
2362 DESC_HDR_MODE0_MDEU_SHA1,
2363 },
2364 { .type = CRYPTO_ALG_TYPE_AHASH,
2365 .alg.hash = {
2366 .halg.digestsize = SHA224_DIGEST_SIZE,
2367 .halg.base = {
2368 .cra_name = "sha224",
2369 .cra_driver_name = "sha224-talitos",
2370 .cra_blocksize = SHA224_BLOCK_SIZE,
2371 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2372 CRYPTO_ALG_ASYNC,
2373 }
2374 },
2375 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2376 DESC_HDR_SEL0_MDEUA |
2377 DESC_HDR_MODE0_MDEU_SHA224,
2378 },
2379 { .type = CRYPTO_ALG_TYPE_AHASH,
2380 .alg.hash = {
2381 .halg.digestsize = SHA256_DIGEST_SIZE,
2382 .halg.base = {
2383 .cra_name = "sha256",
2384 .cra_driver_name = "sha256-talitos",
2385 .cra_blocksize = SHA256_BLOCK_SIZE,
2386 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2387 CRYPTO_ALG_ASYNC,
2388 }
2389 },
2390 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2391 DESC_HDR_SEL0_MDEUA |
2392 DESC_HDR_MODE0_MDEU_SHA256,
2393 },
2394 { .type = CRYPTO_ALG_TYPE_AHASH,
2395 .alg.hash = {
2396 .halg.digestsize = SHA384_DIGEST_SIZE,
2397 .halg.base = {
2398 .cra_name = "sha384",
2399 .cra_driver_name = "sha384-talitos",
2400 .cra_blocksize = SHA384_BLOCK_SIZE,
2401 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2402 CRYPTO_ALG_ASYNC,
2403 }
2404 },
2405 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2406 DESC_HDR_SEL0_MDEUB |
2407 DESC_HDR_MODE0_MDEUB_SHA384,
2408 },
2409 { .type = CRYPTO_ALG_TYPE_AHASH,
2410 .alg.hash = {
2411 .halg.digestsize = SHA512_DIGEST_SIZE,
2412 .halg.base = {
2413 .cra_name = "sha512",
2414 .cra_driver_name = "sha512-talitos",
2415 .cra_blocksize = SHA512_BLOCK_SIZE,
2416 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2417 CRYPTO_ALG_ASYNC,
2418 }
2419 },
2420 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2421 DESC_HDR_SEL0_MDEUB |
2422 DESC_HDR_MODE0_MDEUB_SHA512,
2423 },
2424 { .type = CRYPTO_ALG_TYPE_AHASH,
2425 .alg.hash = {
2426 .halg.digestsize = MD5_DIGEST_SIZE,
2427 .halg.base = {
2428 .cra_name = "hmac(md5)",
2429 .cra_driver_name = "hmac-md5-talitos",
2430 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2431 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2432 CRYPTO_ALG_ASYNC,
2433 }
2434 },
2435 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2436 DESC_HDR_SEL0_MDEUA |
2437 DESC_HDR_MODE0_MDEU_MD5,
2438 },
2439 { .type = CRYPTO_ALG_TYPE_AHASH,
2440 .alg.hash = {
2441 .halg.digestsize = SHA1_DIGEST_SIZE,
2442 .halg.base = {
2443 .cra_name = "hmac(sha1)",
2444 .cra_driver_name = "hmac-sha1-talitos",
2445 .cra_blocksize = SHA1_BLOCK_SIZE,
2446 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2447 CRYPTO_ALG_ASYNC,
2448 }
2449 },
2450 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2451 DESC_HDR_SEL0_MDEUA |
2452 DESC_HDR_MODE0_MDEU_SHA1,
2453 },
2454 { .type = CRYPTO_ALG_TYPE_AHASH,
2455 .alg.hash = {
2456 .halg.digestsize = SHA224_DIGEST_SIZE,
2457 .halg.base = {
2458 .cra_name = "hmac(sha224)",
2459 .cra_driver_name = "hmac-sha224-talitos",
2460 .cra_blocksize = SHA224_BLOCK_SIZE,
2461 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2462 CRYPTO_ALG_ASYNC,
2463 }
2464 },
2465 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2466 DESC_HDR_SEL0_MDEUA |
2467 DESC_HDR_MODE0_MDEU_SHA224,
2468 },
2469 { .type = CRYPTO_ALG_TYPE_AHASH,
2470 .alg.hash = {
2471 .halg.digestsize = SHA256_DIGEST_SIZE,
2472 .halg.base = {
2473 .cra_name = "hmac(sha256)",
2474 .cra_driver_name = "hmac-sha256-talitos",
2475 .cra_blocksize = SHA256_BLOCK_SIZE,
2476 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2477 CRYPTO_ALG_ASYNC,
2478 }
2479 },
2480 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2481 DESC_HDR_SEL0_MDEUA |
2482 DESC_HDR_MODE0_MDEU_SHA256,
2483 },
2484 { .type = CRYPTO_ALG_TYPE_AHASH,
2485 .alg.hash = {
2486 .halg.digestsize = SHA384_DIGEST_SIZE,
2487 .halg.base = {
2488 .cra_name = "hmac(sha384)",
2489 .cra_driver_name = "hmac-sha384-talitos",
2490 .cra_blocksize = SHA384_BLOCK_SIZE,
2491 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2492 CRYPTO_ALG_ASYNC,
2493 }
2494 },
2495 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2496 DESC_HDR_SEL0_MDEUB |
2497 DESC_HDR_MODE0_MDEUB_SHA384,
2498 },
2499 { .type = CRYPTO_ALG_TYPE_AHASH,
2500 .alg.hash = {
2501 .halg.digestsize = SHA512_DIGEST_SIZE,
2502 .halg.base = {
2503 .cra_name = "hmac(sha512)",
2504 .cra_driver_name = "hmac-sha512-talitos",
2505 .cra_blocksize = SHA512_BLOCK_SIZE,
2506 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2507 CRYPTO_ALG_ASYNC,
2508 }
2509 },
2510 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2511 DESC_HDR_SEL0_MDEUB |
2512 DESC_HDR_MODE0_MDEUB_SHA512,
2513 }
2514 };
2515
2516 struct talitos_crypto_alg {
2517 struct list_head entry;
2518 struct device *dev;
2519 struct talitos_alg_template algt;
2520 };
2521
2522 static int talitos_cra_init(struct crypto_tfm *tfm)
2523 {
2524 struct crypto_alg *alg = tfm->__crt_alg;
2525 struct talitos_crypto_alg *talitos_alg;
2526 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2527 struct talitos_private *priv;
2528
2529 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2530 talitos_alg = container_of(__crypto_ahash_alg(alg),
2531 struct talitos_crypto_alg,
2532 algt.alg.hash);
2533 else
2534 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2535 algt.alg.crypto);
2536
2537 /* update context with ptr to dev */
2538 ctx->dev = talitos_alg->dev;
2539
2540 /* assign SEC channel to tfm in round-robin fashion */
2541 priv = dev_get_drvdata(ctx->dev);
2542 ctx->ch = atomic_inc_return(&priv->last_chan) &
2543 (priv->num_channels - 1);
2544
2545 /* copy descriptor header template value */
2546 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2547
2548 /* select done notification */
2549 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2550
2551 return 0;
2552 }
2553
2554 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2555 {
2556 talitos_cra_init(crypto_aead_tfm(tfm));
2557 return 0;
2558 }
2559
2560 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2561 {
2562 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2563
2564 talitos_cra_init(tfm);
2565
2566 ctx->keylen = 0;
2567 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2568 sizeof(struct talitos_ahash_req_ctx));
2569
2570 return 0;
2571 }
2572
2573 /*
2574 * given the alg's descriptor header template, determine whether descriptor
2575 * type and primary/secondary execution units required match the hw
2576 * capabilities description provided in the device tree node.
2577 */
2578 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2579 {
2580 struct talitos_private *priv = dev_get_drvdata(dev);
2581 int ret;
2582
2583 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2584 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2585
2586 if (SECONDARY_EU(desc_hdr_template))
2587 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2588 & priv->exec_units);
2589
2590 return ret;
2591 }
2592
2593 static int talitos_remove(struct platform_device *ofdev)
2594 {
2595 struct device *dev = &ofdev->dev;
2596 struct talitos_private *priv = dev_get_drvdata(dev);
2597 struct talitos_crypto_alg *t_alg, *n;
2598 int i;
2599
2600 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2601 switch (t_alg->algt.type) {
2602 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2603 break;
2604 case CRYPTO_ALG_TYPE_AEAD:
2605 crypto_unregister_aead(&t_alg->algt.alg.aead);
2606 case CRYPTO_ALG_TYPE_AHASH:
2607 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2608 break;
2609 }
2610 list_del(&t_alg->entry);
2611 kfree(t_alg);
2612 }
2613
2614 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2615 talitos_unregister_rng(dev);
2616
2617 for (i = 0; priv->chan && i < priv->num_channels; i++)
2618 kfree(priv->chan[i].fifo);
2619
2620 kfree(priv->chan);
2621
2622 for (i = 0; i < 2; i++)
2623 if (priv->irq[i]) {
2624 free_irq(priv->irq[i], dev);
2625 irq_dispose_mapping(priv->irq[i]);
2626 }
2627
2628 tasklet_kill(&priv->done_task[0]);
2629 if (priv->irq[1])
2630 tasklet_kill(&priv->done_task[1]);
2631
2632 iounmap(priv->reg);
2633
2634 kfree(priv);
2635
2636 return 0;
2637 }
2638
2639 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2640 struct talitos_alg_template
2641 *template)
2642 {
2643 struct talitos_private *priv = dev_get_drvdata(dev);
2644 struct talitos_crypto_alg *t_alg;
2645 struct crypto_alg *alg;
2646
2647 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2648 if (!t_alg)
2649 return ERR_PTR(-ENOMEM);
2650
2651 t_alg->algt = *template;
2652
2653 switch (t_alg->algt.type) {
2654 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2655 alg = &t_alg->algt.alg.crypto;
2656 alg->cra_init = talitos_cra_init;
2657 alg->cra_type = &crypto_ablkcipher_type;
2658 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2659 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2660 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2661 alg->cra_ablkcipher.geniv = "eseqiv";
2662 break;
2663 case CRYPTO_ALG_TYPE_AEAD:
2664 alg = &t_alg->algt.alg.aead.base;
2665 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
2666 t_alg->algt.alg.aead.setkey = aead_setkey;
2667 t_alg->algt.alg.aead.encrypt = aead_encrypt;
2668 t_alg->algt.alg.aead.decrypt = aead_decrypt;
2669 break;
2670 case CRYPTO_ALG_TYPE_AHASH:
2671 alg = &t_alg->algt.alg.hash.halg.base;
2672 alg->cra_init = talitos_cra_init_ahash;
2673 alg->cra_type = &crypto_ahash_type;
2674 t_alg->algt.alg.hash.init = ahash_init;
2675 t_alg->algt.alg.hash.update = ahash_update;
2676 t_alg->algt.alg.hash.final = ahash_final;
2677 t_alg->algt.alg.hash.finup = ahash_finup;
2678 t_alg->algt.alg.hash.digest = ahash_digest;
2679 t_alg->algt.alg.hash.setkey = ahash_setkey;
2680
2681 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2682 !strncmp(alg->cra_name, "hmac", 4)) {
2683 kfree(t_alg);
2684 return ERR_PTR(-ENOTSUPP);
2685 }
2686 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2687 (!strcmp(alg->cra_name, "sha224") ||
2688 !strcmp(alg->cra_name, "hmac(sha224)"))) {
2689 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2690 t_alg->algt.desc_hdr_template =
2691 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2692 DESC_HDR_SEL0_MDEUA |
2693 DESC_HDR_MODE0_MDEU_SHA256;
2694 }
2695 break;
2696 default:
2697 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2698 kfree(t_alg);
2699 return ERR_PTR(-EINVAL);
2700 }
2701
2702 alg->cra_module = THIS_MODULE;
2703 alg->cra_priority = TALITOS_CRA_PRIORITY;
2704 alg->cra_alignmask = 0;
2705 alg->cra_ctxsize = sizeof(struct talitos_ctx);
2706 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2707
2708 t_alg->dev = dev;
2709
2710 return t_alg;
2711 }
2712
2713 static int talitos_probe_irq(struct platform_device *ofdev)
2714 {
2715 struct device *dev = &ofdev->dev;
2716 struct device_node *np = ofdev->dev.of_node;
2717 struct talitos_private *priv = dev_get_drvdata(dev);
2718 int err;
2719 bool is_sec1 = has_ftr_sec1(priv);
2720
2721 priv->irq[0] = irq_of_parse_and_map(np, 0);
2722 if (!priv->irq[0]) {
2723 dev_err(dev, "failed to map irq\n");
2724 return -EINVAL;
2725 }
2726 if (is_sec1) {
2727 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2728 dev_driver_string(dev), dev);
2729 goto primary_out;
2730 }
2731
2732 priv->irq[1] = irq_of_parse_and_map(np, 1);
2733
2734 /* get the primary irq line */
2735 if (!priv->irq[1]) {
2736 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
2737 dev_driver_string(dev), dev);
2738 goto primary_out;
2739 }
2740
2741 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
2742 dev_driver_string(dev), dev);
2743 if (err)
2744 goto primary_out;
2745
2746 /* get the secondary irq line */
2747 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
2748 dev_driver_string(dev), dev);
2749 if (err) {
2750 dev_err(dev, "failed to request secondary irq\n");
2751 irq_dispose_mapping(priv->irq[1]);
2752 priv->irq[1] = 0;
2753 }
2754
2755 return err;
2756
2757 primary_out:
2758 if (err) {
2759 dev_err(dev, "failed to request primary irq\n");
2760 irq_dispose_mapping(priv->irq[0]);
2761 priv->irq[0] = 0;
2762 }
2763
2764 return err;
2765 }
2766
2767 static int talitos_probe(struct platform_device *ofdev)
2768 {
2769 struct device *dev = &ofdev->dev;
2770 struct device_node *np = ofdev->dev.of_node;
2771 struct talitos_private *priv;
2772 const unsigned int *prop;
2773 int i, err;
2774 int stride;
2775
2776 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2777 if (!priv)
2778 return -ENOMEM;
2779
2780 INIT_LIST_HEAD(&priv->alg_list);
2781
2782 dev_set_drvdata(dev, priv);
2783
2784 priv->ofdev = ofdev;
2785
2786 spin_lock_init(&priv->reg_lock);
2787
2788 priv->reg = of_iomap(np, 0);
2789 if (!priv->reg) {
2790 dev_err(dev, "failed to of_iomap\n");
2791 err = -ENOMEM;
2792 goto err_out;
2793 }
2794
2795 /* get SEC version capabilities from device tree */
2796 prop = of_get_property(np, "fsl,num-channels", NULL);
2797 if (prop)
2798 priv->num_channels = *prop;
2799
2800 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2801 if (prop)
2802 priv->chfifo_len = *prop;
2803
2804 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2805 if (prop)
2806 priv->exec_units = *prop;
2807
2808 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2809 if (prop)
2810 priv->desc_types = *prop;
2811
2812 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2813 !priv->exec_units || !priv->desc_types) {
2814 dev_err(dev, "invalid property data in device tree node\n");
2815 err = -EINVAL;
2816 goto err_out;
2817 }
2818
2819 if (of_device_is_compatible(np, "fsl,sec3.0"))
2820 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2821
2822 if (of_device_is_compatible(np, "fsl,sec2.1"))
2823 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2824 TALITOS_FTR_SHA224_HWINIT |
2825 TALITOS_FTR_HMAC_OK;
2826
2827 if (of_device_is_compatible(np, "fsl,sec1.0"))
2828 priv->features |= TALITOS_FTR_SEC1;
2829
2830 if (of_device_is_compatible(np, "fsl,sec1.2")) {
2831 priv->reg_deu = priv->reg + TALITOS12_DEU;
2832 priv->reg_aesu = priv->reg + TALITOS12_AESU;
2833 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
2834 stride = TALITOS1_CH_STRIDE;
2835 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
2836 priv->reg_deu = priv->reg + TALITOS10_DEU;
2837 priv->reg_aesu = priv->reg + TALITOS10_AESU;
2838 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
2839 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
2840 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
2841 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
2842 stride = TALITOS1_CH_STRIDE;
2843 } else {
2844 priv->reg_deu = priv->reg + TALITOS2_DEU;
2845 priv->reg_aesu = priv->reg + TALITOS2_AESU;
2846 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
2847 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
2848 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
2849 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
2850 priv->reg_keu = priv->reg + TALITOS2_KEU;
2851 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
2852 stride = TALITOS2_CH_STRIDE;
2853 }
2854
2855 err = talitos_probe_irq(ofdev);
2856 if (err)
2857 goto err_out;
2858
2859 if (of_device_is_compatible(np, "fsl,sec1.0")) {
2860 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
2861 (unsigned long)dev);
2862 } else {
2863 if (!priv->irq[1]) {
2864 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
2865 (unsigned long)dev);
2866 } else {
2867 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
2868 (unsigned long)dev);
2869 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
2870 (unsigned long)dev);
2871 }
2872 }
2873
2874 priv->chan = kzalloc(sizeof(struct talitos_channel) *
2875 priv->num_channels, GFP_KERNEL);
2876 if (!priv->chan) {
2877 dev_err(dev, "failed to allocate channel management space\n");
2878 err = -ENOMEM;
2879 goto err_out;
2880 }
2881
2882 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2883
2884 for (i = 0; i < priv->num_channels; i++) {
2885 priv->chan[i].reg = priv->reg + stride * (i + 1);
2886 if (!priv->irq[1] || !(i & 1))
2887 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2888
2889 spin_lock_init(&priv->chan[i].head_lock);
2890 spin_lock_init(&priv->chan[i].tail_lock);
2891
2892 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2893 priv->fifo_len, GFP_KERNEL);
2894 if (!priv->chan[i].fifo) {
2895 dev_err(dev, "failed to allocate request fifo %d\n", i);
2896 err = -ENOMEM;
2897 goto err_out;
2898 }
2899
2900 atomic_set(&priv->chan[i].submit_count,
2901 -(priv->chfifo_len - 1));
2902 }
2903
2904 dma_set_mask(dev, DMA_BIT_MASK(36));
2905
2906 /* reset and initialize the h/w */
2907 err = init_device(dev);
2908 if (err) {
2909 dev_err(dev, "failed to initialize device\n");
2910 goto err_out;
2911 }
2912
2913 /* register the RNG, if available */
2914 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
2915 err = talitos_register_rng(dev);
2916 if (err) {
2917 dev_err(dev, "failed to register hwrng: %d\n", err);
2918 goto err_out;
2919 } else
2920 dev_info(dev, "hwrng\n");
2921 }
2922
2923 /* register crypto algorithms the device supports */
2924 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2925 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
2926 struct talitos_crypto_alg *t_alg;
2927 struct crypto_alg *alg = NULL;
2928
2929 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
2930 if (IS_ERR(t_alg)) {
2931 err = PTR_ERR(t_alg);
2932 if (err == -ENOTSUPP)
2933 continue;
2934 goto err_out;
2935 }
2936
2937 switch (t_alg->algt.type) {
2938 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2939 err = crypto_register_alg(
2940 &t_alg->algt.alg.crypto);
2941 alg = &t_alg->algt.alg.crypto;
2942 break;
2943
2944 case CRYPTO_ALG_TYPE_AEAD:
2945 err = crypto_register_aead(
2946 &t_alg->algt.alg.aead);
2947 alg = &t_alg->algt.alg.aead.base;
2948 break;
2949
2950 case CRYPTO_ALG_TYPE_AHASH:
2951 err = crypto_register_ahash(
2952 &t_alg->algt.alg.hash);
2953 alg = &t_alg->algt.alg.hash.halg.base;
2954 break;
2955 }
2956 if (err) {
2957 dev_err(dev, "%s alg registration failed\n",
2958 alg->cra_driver_name);
2959 kfree(t_alg);
2960 } else
2961 list_add_tail(&t_alg->entry, &priv->alg_list);
2962 }
2963 }
2964 if (!list_empty(&priv->alg_list))
2965 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
2966 (char *)of_get_property(np, "compatible", NULL));
2967
2968 return 0;
2969
2970 err_out:
2971 talitos_remove(ofdev);
2972
2973 return err;
2974 }
2975
2976 static const struct of_device_id talitos_match[] = {
2977 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
2978 {
2979 .compatible = "fsl,sec1.0",
2980 },
2981 #endif
2982 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
2983 {
2984 .compatible = "fsl,sec2.0",
2985 },
2986 #endif
2987 {},
2988 };
2989 MODULE_DEVICE_TABLE(of, talitos_match);
2990
2991 static struct platform_driver talitos_driver = {
2992 .driver = {
2993 .name = "talitos",
2994 .of_match_table = talitos_match,
2995 },
2996 .probe = talitos_probe,
2997 .remove = talitos_remove,
2998 };
2999
3000 module_platform_driver(talitos_driver);
3001
3002 MODULE_LICENSE("GPL");
3003 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3004 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
Michael's Linux tree.