1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Cryptographic API for algorithms (i.e., low-level API).
5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
8 #include <crypto/algapi.h>
10 #include <linux/errno.h>
11 #include <linux/fips.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/list.h>
15 #include <linux/module.h>
16 #include <linux/rtnetlink.h>
17 #include <linux/slab.h>
18 #include <linux/string.h>
22 static LIST_HEAD(crypto_template_list
);
24 static inline int crypto_set_driver_name(struct crypto_alg
*alg
)
26 static const char suffix
[] = "-generic";
27 char *driver_name
= alg
->cra_driver_name
;
33 len
= strlcpy(driver_name
, alg
->cra_name
, CRYPTO_MAX_ALG_NAME
);
34 if (len
+ sizeof(suffix
) > CRYPTO_MAX_ALG_NAME
)
37 memcpy(driver_name
+ len
, suffix
, sizeof(suffix
));
41 static inline void crypto_check_module_sig(struct module
*mod
)
43 if (fips_enabled
&& mod
&& !module_sig_ok(mod
))
44 panic("Module %s signature verification failed in FIPS mode\n",
48 static int crypto_check_alg(struct crypto_alg
*alg
)
50 crypto_check_module_sig(alg
->cra_module
);
52 if (alg
->cra_alignmask
& (alg
->cra_alignmask
+ 1))
55 /* General maximums for all algs. */
56 if (alg
->cra_alignmask
> MAX_ALGAPI_ALIGNMASK
)
59 if (alg
->cra_blocksize
> MAX_ALGAPI_BLOCKSIZE
)
62 /* Lower maximums for specific alg types. */
63 if (!alg
->cra_type
&& (alg
->cra_flags
& CRYPTO_ALG_TYPE_MASK
) ==
64 CRYPTO_ALG_TYPE_CIPHER
) {
65 if (alg
->cra_alignmask
> MAX_CIPHER_ALIGNMASK
)
68 if (alg
->cra_blocksize
> MAX_CIPHER_BLOCKSIZE
)
72 if (alg
->cra_priority
< 0)
75 refcount_set(&alg
->cra_refcnt
, 1);
77 return crypto_set_driver_name(alg
);
80 static void crypto_free_instance(struct crypto_instance
*inst
)
82 if (!inst
->alg
.cra_type
->free
) {
83 inst
->tmpl
->free(inst
);
87 inst
->alg
.cra_type
->free(inst
);
90 static void crypto_destroy_instance(struct crypto_alg
*alg
)
92 struct crypto_instance
*inst
= (void *)alg
;
93 struct crypto_template
*tmpl
= inst
->tmpl
;
95 crypto_free_instance(inst
);
96 crypto_tmpl_put(tmpl
);
99 static struct list_head
*crypto_more_spawns(struct crypto_alg
*alg
,
100 struct list_head
*stack
,
101 struct list_head
*top
,
102 struct list_head
*secondary_spawns
)
104 struct crypto_spawn
*spawn
, *n
;
106 spawn
= list_first_entry_or_null(stack
, struct crypto_spawn
, list
);
110 n
= list_next_entry(spawn
, list
);
112 if (spawn
->alg
&& &n
->list
!= stack
&& !n
->alg
)
113 n
->alg
= (n
->list
.next
== stack
) ? alg
:
114 &list_next_entry(n
, list
)->inst
->alg
;
116 list_move(&spawn
->list
, secondary_spawns
);
118 return &n
->list
== stack
? top
: &n
->inst
->alg
.cra_users
;
121 static void crypto_remove_instance(struct crypto_instance
*inst
,
122 struct list_head
*list
)
124 struct crypto_template
*tmpl
= inst
->tmpl
;
126 if (crypto_is_dead(&inst
->alg
))
129 inst
->alg
.cra_flags
|= CRYPTO_ALG_DEAD
;
130 if (hlist_unhashed(&inst
->list
))
133 if (!tmpl
|| !crypto_tmpl_get(tmpl
))
136 list_move(&inst
->alg
.cra_list
, list
);
137 hlist_del(&inst
->list
);
138 inst
->alg
.cra_destroy
= crypto_destroy_instance
;
140 BUG_ON(!list_empty(&inst
->alg
.cra_users
));
143 void crypto_remove_spawns(struct crypto_alg
*alg
, struct list_head
*list
,
144 struct crypto_alg
*nalg
)
146 u32 new_type
= (nalg
?: alg
)->cra_flags
;
147 struct crypto_spawn
*spawn
, *n
;
148 LIST_HEAD(secondary_spawns
);
149 struct list_head
*spawns
;
153 spawns
= &alg
->cra_users
;
154 list_for_each_entry_safe(spawn
, n
, spawns
, list
) {
155 if ((spawn
->alg
->cra_flags
^ new_type
) & spawn
->mask
)
158 list_move(&spawn
->list
, &top
);
163 while (!list_empty(spawns
)) {
164 struct crypto_instance
*inst
;
166 spawn
= list_first_entry(spawns
, struct crypto_spawn
,
170 BUG_ON(&inst
->alg
== alg
);
172 list_move(&spawn
->list
, &stack
);
174 if (&inst
->alg
== nalg
)
178 spawns
= &inst
->alg
.cra_users
;
181 * We may encounter an unregistered instance here, since
182 * an instance's spawns are set up prior to the instance
183 * being registered. An unregistered instance will have
184 * NULL ->cra_users.next, since ->cra_users isn't
185 * properly initialized until registration. But an
186 * unregistered instance cannot have any users, so treat
187 * it the same as ->cra_users being empty.
189 if (spawns
->next
== NULL
)
192 } while ((spawns
= crypto_more_spawns(alg
, &stack
, &top
,
193 &secondary_spawns
)));
195 list_for_each_entry_safe(spawn
, n
, &secondary_spawns
, list
) {
197 list_move(&spawn
->list
, &spawn
->alg
->cra_users
);
199 crypto_remove_instance(spawn
->inst
, list
);
202 EXPORT_SYMBOL_GPL(crypto_remove_spawns
);
204 static struct crypto_larval
*__crypto_register_alg(struct crypto_alg
*alg
)
206 struct crypto_alg
*q
;
207 struct crypto_larval
*larval
;
210 if (crypto_is_dead(alg
))
213 INIT_LIST_HEAD(&alg
->cra_users
);
216 alg
->cra_flags
&= ~CRYPTO_ALG_TESTED
;
220 list_for_each_entry(q
, &crypto_alg_list
, cra_list
) {
224 if (crypto_is_moribund(q
))
227 if (crypto_is_larval(q
)) {
228 if (!strcmp(alg
->cra_driver_name
, q
->cra_driver_name
))
233 if (!strcmp(q
->cra_driver_name
, alg
->cra_name
) ||
234 !strcmp(q
->cra_name
, alg
->cra_driver_name
))
238 larval
= crypto_larval_alloc(alg
->cra_name
,
239 alg
->cra_flags
| CRYPTO_ALG_TESTED
, 0);
244 larval
->adult
= crypto_mod_get(alg
);
248 refcount_set(&larval
->alg
.cra_refcnt
, 1);
249 memcpy(larval
->alg
.cra_driver_name
, alg
->cra_driver_name
,
250 CRYPTO_MAX_ALG_NAME
);
251 larval
->alg
.cra_priority
= alg
->cra_priority
;
253 list_add(&alg
->cra_list
, &crypto_alg_list
);
254 list_add(&larval
->alg
.cra_list
, &crypto_alg_list
);
256 crypto_stats_init(alg
);
264 larval
= ERR_PTR(ret
);
268 void crypto_alg_tested(const char *name
, int err
)
270 struct crypto_larval
*test
;
271 struct crypto_alg
*alg
;
272 struct crypto_alg
*q
;
275 down_write(&crypto_alg_sem
);
276 list_for_each_entry(q
, &crypto_alg_list
, cra_list
) {
277 if (crypto_is_moribund(q
) || !crypto_is_larval(q
))
280 test
= (struct crypto_larval
*)q
;
282 if (!strcmp(q
->cra_driver_name
, name
))
286 pr_err("alg: Unexpected test result for %s: %d\n", name
, err
);
290 q
->cra_flags
|= CRYPTO_ALG_DEAD
;
292 if (err
|| list_empty(&alg
->cra_list
))
295 alg
->cra_flags
|= CRYPTO_ALG_TESTED
;
297 list_for_each_entry(q
, &crypto_alg_list
, cra_list
) {
301 if (crypto_is_moribund(q
))
304 if (crypto_is_larval(q
)) {
305 struct crypto_larval
*larval
= (void *)q
;
308 * Check to see if either our generic name or
309 * specific name can satisfy the name requested
310 * by the larval entry q.
312 if (strcmp(alg
->cra_name
, q
->cra_name
) &&
313 strcmp(alg
->cra_driver_name
, q
->cra_name
))
318 if ((q
->cra_flags
^ alg
->cra_flags
) & larval
->mask
)
320 if (!crypto_mod_get(alg
))
327 if (strcmp(alg
->cra_name
, q
->cra_name
))
330 if (strcmp(alg
->cra_driver_name
, q
->cra_driver_name
) &&
331 q
->cra_priority
> alg
->cra_priority
)
334 crypto_remove_spawns(q
, &list
, alg
);
338 complete_all(&test
->completion
);
341 up_write(&crypto_alg_sem
);
343 crypto_remove_final(&list
);
345 EXPORT_SYMBOL_GPL(crypto_alg_tested
);
347 void crypto_remove_final(struct list_head
*list
)
349 struct crypto_alg
*alg
;
350 struct crypto_alg
*n
;
352 list_for_each_entry_safe(alg
, n
, list
, cra_list
) {
353 list_del_init(&alg
->cra_list
);
357 EXPORT_SYMBOL_GPL(crypto_remove_final
);
359 static void crypto_wait_for_test(struct crypto_larval
*larval
)
363 err
= crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER
, larval
->adult
);
364 if (err
!= NOTIFY_STOP
) {
365 if (WARN_ON(err
!= NOTIFY_DONE
))
367 crypto_alg_tested(larval
->alg
.cra_driver_name
, 0);
370 err
= wait_for_completion_killable(&larval
->completion
);
373 crypto_probing_notify(CRYPTO_MSG_ALG_LOADED
, larval
);
376 crypto_larval_kill(&larval
->alg
);
379 int crypto_register_alg(struct crypto_alg
*alg
)
381 struct crypto_larval
*larval
;
384 alg
->cra_flags
&= ~CRYPTO_ALG_DEAD
;
385 err
= crypto_check_alg(alg
);
389 down_write(&crypto_alg_sem
);
390 larval
= __crypto_register_alg(alg
);
391 up_write(&crypto_alg_sem
);
394 return PTR_ERR(larval
);
396 crypto_wait_for_test(larval
);
399 EXPORT_SYMBOL_GPL(crypto_register_alg
);
401 static int crypto_remove_alg(struct crypto_alg
*alg
, struct list_head
*list
)
403 if (unlikely(list_empty(&alg
->cra_list
)))
406 alg
->cra_flags
|= CRYPTO_ALG_DEAD
;
408 list_del_init(&alg
->cra_list
);
409 crypto_remove_spawns(alg
, list
, NULL
);
414 int crypto_unregister_alg(struct crypto_alg
*alg
)
419 down_write(&crypto_alg_sem
);
420 ret
= crypto_remove_alg(alg
, &list
);
421 up_write(&crypto_alg_sem
);
426 BUG_ON(refcount_read(&alg
->cra_refcnt
) != 1);
427 if (alg
->cra_destroy
)
428 alg
->cra_destroy(alg
);
430 crypto_remove_final(&list
);
433 EXPORT_SYMBOL_GPL(crypto_unregister_alg
);
435 int crypto_register_algs(struct crypto_alg
*algs
, int count
)
439 for (i
= 0; i
< count
; i
++) {
440 ret
= crypto_register_alg(&algs
[i
]);
448 for (--i
; i
>= 0; --i
)
449 crypto_unregister_alg(&algs
[i
]);
453 EXPORT_SYMBOL_GPL(crypto_register_algs
);
455 int crypto_unregister_algs(struct crypto_alg
*algs
, int count
)
459 for (i
= 0; i
< count
; i
++) {
460 ret
= crypto_unregister_alg(&algs
[i
]);
462 pr_err("Failed to unregister %s %s: %d\n",
463 algs
[i
].cra_driver_name
, algs
[i
].cra_name
, ret
);
468 EXPORT_SYMBOL_GPL(crypto_unregister_algs
);
470 int crypto_register_template(struct crypto_template
*tmpl
)
472 struct crypto_template
*q
;
475 down_write(&crypto_alg_sem
);
477 crypto_check_module_sig(tmpl
->module
);
479 list_for_each_entry(q
, &crypto_template_list
, list
) {
484 list_add(&tmpl
->list
, &crypto_template_list
);
487 up_write(&crypto_alg_sem
);
490 EXPORT_SYMBOL_GPL(crypto_register_template
);
492 int crypto_register_templates(struct crypto_template
*tmpls
, int count
)
496 for (i
= 0; i
< count
; i
++) {
497 err
= crypto_register_template(&tmpls
[i
]);
504 for (--i
; i
>= 0; --i
)
505 crypto_unregister_template(&tmpls
[i
]);
508 EXPORT_SYMBOL_GPL(crypto_register_templates
);
510 void crypto_unregister_template(struct crypto_template
*tmpl
)
512 struct crypto_instance
*inst
;
513 struct hlist_node
*n
;
514 struct hlist_head
*list
;
517 down_write(&crypto_alg_sem
);
519 BUG_ON(list_empty(&tmpl
->list
));
520 list_del_init(&tmpl
->list
);
522 list
= &tmpl
->instances
;
523 hlist_for_each_entry(inst
, list
, list
) {
524 int err
= crypto_remove_alg(&inst
->alg
, &users
);
529 up_write(&crypto_alg_sem
);
531 hlist_for_each_entry_safe(inst
, n
, list
, list
) {
532 BUG_ON(refcount_read(&inst
->alg
.cra_refcnt
) != 1);
533 crypto_free_instance(inst
);
535 crypto_remove_final(&users
);
537 EXPORT_SYMBOL_GPL(crypto_unregister_template
);
539 void crypto_unregister_templates(struct crypto_template
*tmpls
, int count
)
543 for (i
= count
- 1; i
>= 0; --i
)
544 crypto_unregister_template(&tmpls
[i
]);
546 EXPORT_SYMBOL_GPL(crypto_unregister_templates
);
548 static struct crypto_template
*__crypto_lookup_template(const char *name
)
550 struct crypto_template
*q
, *tmpl
= NULL
;
552 down_read(&crypto_alg_sem
);
553 list_for_each_entry(q
, &crypto_template_list
, list
) {
554 if (strcmp(q
->name
, name
))
556 if (unlikely(!crypto_tmpl_get(q
)))
562 up_read(&crypto_alg_sem
);
567 struct crypto_template
*crypto_lookup_template(const char *name
)
569 return try_then_request_module(__crypto_lookup_template(name
),
572 EXPORT_SYMBOL_GPL(crypto_lookup_template
);
574 int crypto_register_instance(struct crypto_template
*tmpl
,
575 struct crypto_instance
*inst
)
577 struct crypto_larval
*larval
;
580 err
= crypto_check_alg(&inst
->alg
);
584 inst
->alg
.cra_module
= tmpl
->module
;
585 inst
->alg
.cra_flags
|= CRYPTO_ALG_INSTANCE
;
587 down_write(&crypto_alg_sem
);
589 larval
= __crypto_register_alg(&inst
->alg
);
593 hlist_add_head(&inst
->list
, &tmpl
->instances
);
597 up_write(&crypto_alg_sem
);
599 err
= PTR_ERR(larval
);
603 crypto_wait_for_test(larval
);
609 EXPORT_SYMBOL_GPL(crypto_register_instance
);
611 int crypto_unregister_instance(struct crypto_instance
*inst
)
615 down_write(&crypto_alg_sem
);
617 crypto_remove_spawns(&inst
->alg
, &list
, NULL
);
618 crypto_remove_instance(inst
, &list
);
620 up_write(&crypto_alg_sem
);
622 crypto_remove_final(&list
);
626 EXPORT_SYMBOL_GPL(crypto_unregister_instance
);
628 int crypto_init_spawn(struct crypto_spawn
*spawn
, struct crypto_alg
*alg
,
629 struct crypto_instance
*inst
, u32 mask
)
633 if (WARN_ON_ONCE(inst
== NULL
))
639 down_write(&crypto_alg_sem
);
640 if (!crypto_is_moribund(alg
)) {
641 list_add(&spawn
->list
, &alg
->cra_users
);
645 up_write(&crypto_alg_sem
);
649 EXPORT_SYMBOL_GPL(crypto_init_spawn
);
651 int crypto_init_spawn2(struct crypto_spawn
*spawn
, struct crypto_alg
*alg
,
652 struct crypto_instance
*inst
,
653 const struct crypto_type
*frontend
)
657 if ((alg
->cra_flags
^ frontend
->type
) & frontend
->maskset
)
660 spawn
->frontend
= frontend
;
661 err
= crypto_init_spawn(spawn
, alg
, inst
, frontend
->maskset
);
666 EXPORT_SYMBOL_GPL(crypto_init_spawn2
);
668 int crypto_grab_spawn(struct crypto_spawn
*spawn
, const char *name
,
671 struct crypto_alg
*alg
;
674 alg
= crypto_find_alg(name
, spawn
->frontend
, type
, mask
);
678 err
= crypto_init_spawn(spawn
, alg
, spawn
->inst
, mask
);
682 EXPORT_SYMBOL_GPL(crypto_grab_spawn
);
684 void crypto_drop_spawn(struct crypto_spawn
*spawn
)
689 down_write(&crypto_alg_sem
);
690 list_del(&spawn
->list
);
691 up_write(&crypto_alg_sem
);
693 EXPORT_SYMBOL_GPL(crypto_drop_spawn
);
695 static struct crypto_alg
*crypto_spawn_alg(struct crypto_spawn
*spawn
)
697 struct crypto_alg
*alg
;
698 struct crypto_alg
*alg2
;
700 down_read(&crypto_alg_sem
);
704 alg2
= crypto_mod_get(alg2
);
705 up_read(&crypto_alg_sem
);
709 crypto_shoot_alg(alg
);
710 return ERR_PTR(-EAGAIN
);
716 struct crypto_tfm
*crypto_spawn_tfm(struct crypto_spawn
*spawn
, u32 type
,
719 struct crypto_alg
*alg
;
720 struct crypto_tfm
*tfm
;
722 alg
= crypto_spawn_alg(spawn
);
724 return ERR_CAST(alg
);
726 tfm
= ERR_PTR(-EINVAL
);
727 if (unlikely((alg
->cra_flags
^ type
) & mask
))
730 tfm
= __crypto_alloc_tfm(alg
, type
, mask
);
740 EXPORT_SYMBOL_GPL(crypto_spawn_tfm
);
742 void *crypto_spawn_tfm2(struct crypto_spawn
*spawn
)
744 struct crypto_alg
*alg
;
745 struct crypto_tfm
*tfm
;
747 alg
= crypto_spawn_alg(spawn
);
749 return ERR_CAST(alg
);
751 tfm
= crypto_create_tfm(alg
, spawn
->frontend
);
761 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2
);
763 int crypto_register_notifier(struct notifier_block
*nb
)
765 return blocking_notifier_chain_register(&crypto_chain
, nb
);
767 EXPORT_SYMBOL_GPL(crypto_register_notifier
);
769 int crypto_unregister_notifier(struct notifier_block
*nb
)
771 return blocking_notifier_chain_unregister(&crypto_chain
, nb
);
773 EXPORT_SYMBOL_GPL(crypto_unregister_notifier
);
775 struct crypto_attr_type
*crypto_get_attr_type(struct rtattr
**tb
)
777 struct rtattr
*rta
= tb
[0];
778 struct crypto_attr_type
*algt
;
781 return ERR_PTR(-ENOENT
);
782 if (RTA_PAYLOAD(rta
) < sizeof(*algt
))
783 return ERR_PTR(-EINVAL
);
784 if (rta
->rta_type
!= CRYPTOA_TYPE
)
785 return ERR_PTR(-EINVAL
);
787 algt
= RTA_DATA(rta
);
791 EXPORT_SYMBOL_GPL(crypto_get_attr_type
);
793 int crypto_check_attr_type(struct rtattr
**tb
, u32 type
)
795 struct crypto_attr_type
*algt
;
797 algt
= crypto_get_attr_type(tb
);
799 return PTR_ERR(algt
);
801 if ((algt
->type
^ type
) & algt
->mask
)
806 EXPORT_SYMBOL_GPL(crypto_check_attr_type
);
808 const char *crypto_attr_alg_name(struct rtattr
*rta
)
810 struct crypto_attr_alg
*alga
;
813 return ERR_PTR(-ENOENT
);
814 if (RTA_PAYLOAD(rta
) < sizeof(*alga
))
815 return ERR_PTR(-EINVAL
);
816 if (rta
->rta_type
!= CRYPTOA_ALG
)
817 return ERR_PTR(-EINVAL
);
819 alga
= RTA_DATA(rta
);
820 alga
->name
[CRYPTO_MAX_ALG_NAME
- 1] = 0;
824 EXPORT_SYMBOL_GPL(crypto_attr_alg_name
);
826 struct crypto_alg
*crypto_attr_alg2(struct rtattr
*rta
,
827 const struct crypto_type
*frontend
,
832 name
= crypto_attr_alg_name(rta
);
834 return ERR_CAST(name
);
836 return crypto_find_alg(name
, frontend
, type
, mask
);
838 EXPORT_SYMBOL_GPL(crypto_attr_alg2
);
840 int crypto_attr_u32(struct rtattr
*rta
, u32
*num
)
842 struct crypto_attr_u32
*nu32
;
846 if (RTA_PAYLOAD(rta
) < sizeof(*nu32
))
848 if (rta
->rta_type
!= CRYPTOA_U32
)
851 nu32
= RTA_DATA(rta
);
856 EXPORT_SYMBOL_GPL(crypto_attr_u32
);
858 int crypto_inst_setname(struct crypto_instance
*inst
, const char *name
,
859 struct crypto_alg
*alg
)
861 if (snprintf(inst
->alg
.cra_name
, CRYPTO_MAX_ALG_NAME
, "%s(%s)", name
,
862 alg
->cra_name
) >= CRYPTO_MAX_ALG_NAME
)
863 return -ENAMETOOLONG
;
865 if (snprintf(inst
->alg
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s(%s)",
866 name
, alg
->cra_driver_name
) >= CRYPTO_MAX_ALG_NAME
)
867 return -ENAMETOOLONG
;
871 EXPORT_SYMBOL_GPL(crypto_inst_setname
);
873 void *crypto_alloc_instance(const char *name
, struct crypto_alg
*alg
,
876 struct crypto_instance
*inst
;
880 p
= kzalloc(head
+ sizeof(*inst
) + sizeof(struct crypto_spawn
),
883 return ERR_PTR(-ENOMEM
);
885 inst
= (void *)(p
+ head
);
887 err
= crypto_inst_setname(inst
, name
, alg
);
897 EXPORT_SYMBOL_GPL(crypto_alloc_instance
);
899 void crypto_init_queue(struct crypto_queue
*queue
, unsigned int max_qlen
)
901 INIT_LIST_HEAD(&queue
->list
);
902 queue
->backlog
= &queue
->list
;
904 queue
->max_qlen
= max_qlen
;
906 EXPORT_SYMBOL_GPL(crypto_init_queue
);
908 int crypto_enqueue_request(struct crypto_queue
*queue
,
909 struct crypto_async_request
*request
)
911 int err
= -EINPROGRESS
;
913 if (unlikely(queue
->qlen
>= queue
->max_qlen
)) {
914 if (!(request
->flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)) {
919 if (queue
->backlog
== &queue
->list
)
920 queue
->backlog
= &request
->list
;
924 list_add_tail(&request
->list
, &queue
->list
);
929 EXPORT_SYMBOL_GPL(crypto_enqueue_request
);
931 struct crypto_async_request
*crypto_dequeue_request(struct crypto_queue
*queue
)
933 struct list_head
*request
;
935 if (unlikely(!queue
->qlen
))
940 if (queue
->backlog
!= &queue
->list
)
941 queue
->backlog
= queue
->backlog
->next
;
943 request
= queue
->list
.next
;
946 return list_entry(request
, struct crypto_async_request
, list
);
948 EXPORT_SYMBOL_GPL(crypto_dequeue_request
);
950 int crypto_tfm_in_queue(struct crypto_queue
*queue
, struct crypto_tfm
*tfm
)
952 struct crypto_async_request
*req
;
954 list_for_each_entry(req
, &queue
->list
, list
) {
961 EXPORT_SYMBOL_GPL(crypto_tfm_in_queue
);
963 static inline void crypto_inc_byte(u8
*a
, unsigned int size
)
968 for (; size
; size
--) {
976 void crypto_inc(u8
*a
, unsigned int size
)
978 __be32
*b
= (__be32
*)(a
+ size
);
981 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
) ||
982 IS_ALIGNED((unsigned long)b
, __alignof__(*b
)))
983 for (; size
>= 4; size
-= 4) {
984 c
= be32_to_cpu(*--b
) + 1;
990 crypto_inc_byte(a
, size
);
992 EXPORT_SYMBOL_GPL(crypto_inc
);
994 void __crypto_xor(u8
*dst
, const u8
*src1
, const u8
*src2
, unsigned int len
)
998 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
)) {
999 int size
= sizeof(unsigned long);
1000 int d
= (((unsigned long)dst
^ (unsigned long)src1
) |
1001 ((unsigned long)dst
^ (unsigned long)src2
)) &
1004 relalign
= d
? 1 << __ffs(d
) : size
;
1007 * If we care about alignment, process as many bytes as
1008 * needed to advance dst and src to values whose alignments
1009 * equal their relative alignment. This will allow us to
1010 * process the remainder of the input using optimal strides.
1012 while (((unsigned long)dst
& (relalign
- 1)) && len
> 0) {
1013 *dst
++ = *src1
++ ^ *src2
++;
1018 while (IS_ENABLED(CONFIG_64BIT
) && len
>= 8 && !(relalign
& 7)) {
1019 *(u64
*)dst
= *(u64
*)src1
^ *(u64
*)src2
;
1026 while (len
>= 4 && !(relalign
& 3)) {
1027 *(u32
*)dst
= *(u32
*)src1
^ *(u32
*)src2
;
1034 while (len
>= 2 && !(relalign
& 1)) {
1035 *(u16
*)dst
= *(u16
*)src1
^ *(u16
*)src2
;
1043 *dst
++ = *src1
++ ^ *src2
++;
1045 EXPORT_SYMBOL_GPL(__crypto_xor
);
1047 unsigned int crypto_alg_extsize(struct crypto_alg
*alg
)
1049 return alg
->cra_ctxsize
+
1050 (alg
->cra_alignmask
& ~(crypto_tfm_ctx_alignment() - 1));
1052 EXPORT_SYMBOL_GPL(crypto_alg_extsize
);
1054 int crypto_type_has_alg(const char *name
, const struct crypto_type
*frontend
,
1058 struct crypto_alg
*alg
= crypto_find_alg(name
, frontend
, type
, mask
);
1061 crypto_mod_put(alg
);
1067 EXPORT_SYMBOL_GPL(crypto_type_has_alg
);
1069 #ifdef CONFIG_CRYPTO_STATS
1070 void crypto_stats_init(struct crypto_alg
*alg
)
1072 memset(&alg
->stats
, 0, sizeof(alg
->stats
));
1074 EXPORT_SYMBOL_GPL(crypto_stats_init
);
1076 void crypto_stats_get(struct crypto_alg
*alg
)
1078 crypto_alg_get(alg
);
1080 EXPORT_SYMBOL_GPL(crypto_stats_get
);
1082 void crypto_stats_ablkcipher_encrypt(unsigned int nbytes
, int ret
,
1083 struct crypto_alg
*alg
)
1085 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1086 atomic64_inc(&alg
->stats
.cipher
.err_cnt
);
1088 atomic64_inc(&alg
->stats
.cipher
.encrypt_cnt
);
1089 atomic64_add(nbytes
, &alg
->stats
.cipher
.encrypt_tlen
);
1091 crypto_alg_put(alg
);
1093 EXPORT_SYMBOL_GPL(crypto_stats_ablkcipher_encrypt
);
1095 void crypto_stats_ablkcipher_decrypt(unsigned int nbytes
, int ret
,
1096 struct crypto_alg
*alg
)
1098 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1099 atomic64_inc(&alg
->stats
.cipher
.err_cnt
);
1101 atomic64_inc(&alg
->stats
.cipher
.decrypt_cnt
);
1102 atomic64_add(nbytes
, &alg
->stats
.cipher
.decrypt_tlen
);
1104 crypto_alg_put(alg
);
1106 EXPORT_SYMBOL_GPL(crypto_stats_ablkcipher_decrypt
);
1108 void crypto_stats_aead_encrypt(unsigned int cryptlen
, struct crypto_alg
*alg
,
1111 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1112 atomic64_inc(&alg
->stats
.aead
.err_cnt
);
1114 atomic64_inc(&alg
->stats
.aead
.encrypt_cnt
);
1115 atomic64_add(cryptlen
, &alg
->stats
.aead
.encrypt_tlen
);
1117 crypto_alg_put(alg
);
1119 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt
);
1121 void crypto_stats_aead_decrypt(unsigned int cryptlen
, struct crypto_alg
*alg
,
1124 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1125 atomic64_inc(&alg
->stats
.aead
.err_cnt
);
1127 atomic64_inc(&alg
->stats
.aead
.decrypt_cnt
);
1128 atomic64_add(cryptlen
, &alg
->stats
.aead
.decrypt_tlen
);
1130 crypto_alg_put(alg
);
1132 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt
);
1134 void crypto_stats_akcipher_encrypt(unsigned int src_len
, int ret
,
1135 struct crypto_alg
*alg
)
1137 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1138 atomic64_inc(&alg
->stats
.akcipher
.err_cnt
);
1140 atomic64_inc(&alg
->stats
.akcipher
.encrypt_cnt
);
1141 atomic64_add(src_len
, &alg
->stats
.akcipher
.encrypt_tlen
);
1143 crypto_alg_put(alg
);
1145 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt
);
1147 void crypto_stats_akcipher_decrypt(unsigned int src_len
, int ret
,
1148 struct crypto_alg
*alg
)
1150 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1151 atomic64_inc(&alg
->stats
.akcipher
.err_cnt
);
1153 atomic64_inc(&alg
->stats
.akcipher
.decrypt_cnt
);
1154 atomic64_add(src_len
, &alg
->stats
.akcipher
.decrypt_tlen
);
1156 crypto_alg_put(alg
);
1158 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt
);
1160 void crypto_stats_akcipher_sign(int ret
, struct crypto_alg
*alg
)
1162 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
)
1163 atomic64_inc(&alg
->stats
.akcipher
.err_cnt
);
1165 atomic64_inc(&alg
->stats
.akcipher
.sign_cnt
);
1166 crypto_alg_put(alg
);
1168 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign
);
1170 void crypto_stats_akcipher_verify(int ret
, struct crypto_alg
*alg
)
1172 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
)
1173 atomic64_inc(&alg
->stats
.akcipher
.err_cnt
);
1175 atomic64_inc(&alg
->stats
.akcipher
.verify_cnt
);
1176 crypto_alg_put(alg
);
1178 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify
);
1180 void crypto_stats_compress(unsigned int slen
, int ret
, struct crypto_alg
*alg
)
1182 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1183 atomic64_inc(&alg
->stats
.compress
.err_cnt
);
1185 atomic64_inc(&alg
->stats
.compress
.compress_cnt
);
1186 atomic64_add(slen
, &alg
->stats
.compress
.compress_tlen
);
1188 crypto_alg_put(alg
);
1190 EXPORT_SYMBOL_GPL(crypto_stats_compress
);
1192 void crypto_stats_decompress(unsigned int slen
, int ret
, struct crypto_alg
*alg
)
1194 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1195 atomic64_inc(&alg
->stats
.compress
.err_cnt
);
1197 atomic64_inc(&alg
->stats
.compress
.decompress_cnt
);
1198 atomic64_add(slen
, &alg
->stats
.compress
.decompress_tlen
);
1200 crypto_alg_put(alg
);
1202 EXPORT_SYMBOL_GPL(crypto_stats_decompress
);
1204 void crypto_stats_ahash_update(unsigned int nbytes
, int ret
,
1205 struct crypto_alg
*alg
)
1207 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
)
1208 atomic64_inc(&alg
->stats
.hash
.err_cnt
);
1210 atomic64_add(nbytes
, &alg
->stats
.hash
.hash_tlen
);
1211 crypto_alg_put(alg
);
1213 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update
);
1215 void crypto_stats_ahash_final(unsigned int nbytes
, int ret
,
1216 struct crypto_alg
*alg
)
1218 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1219 atomic64_inc(&alg
->stats
.hash
.err_cnt
);
1221 atomic64_inc(&alg
->stats
.hash
.hash_cnt
);
1222 atomic64_add(nbytes
, &alg
->stats
.hash
.hash_tlen
);
1224 crypto_alg_put(alg
);
1226 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final
);
1228 void crypto_stats_kpp_set_secret(struct crypto_alg
*alg
, int ret
)
1231 atomic64_inc(&alg
->stats
.kpp
.err_cnt
);
1233 atomic64_inc(&alg
->stats
.kpp
.setsecret_cnt
);
1234 crypto_alg_put(alg
);
1236 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret
);
1238 void crypto_stats_kpp_generate_public_key(struct crypto_alg
*alg
, int ret
)
1241 atomic64_inc(&alg
->stats
.kpp
.err_cnt
);
1243 atomic64_inc(&alg
->stats
.kpp
.generate_public_key_cnt
);
1244 crypto_alg_put(alg
);
1246 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key
);
1248 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg
*alg
, int ret
)
1251 atomic64_inc(&alg
->stats
.kpp
.err_cnt
);
1253 atomic64_inc(&alg
->stats
.kpp
.compute_shared_secret_cnt
);
1254 crypto_alg_put(alg
);
1256 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret
);
1258 void crypto_stats_rng_seed(struct crypto_alg
*alg
, int ret
)
1260 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
)
1261 atomic64_inc(&alg
->stats
.rng
.err_cnt
);
1263 atomic64_inc(&alg
->stats
.rng
.seed_cnt
);
1264 crypto_alg_put(alg
);
1266 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed
);
1268 void crypto_stats_rng_generate(struct crypto_alg
*alg
, unsigned int dlen
,
1271 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1272 atomic64_inc(&alg
->stats
.rng
.err_cnt
);
1274 atomic64_inc(&alg
->stats
.rng
.generate_cnt
);
1275 atomic64_add(dlen
, &alg
->stats
.rng
.generate_tlen
);
1277 crypto_alg_put(alg
);
1279 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate
);
1281 void crypto_stats_skcipher_encrypt(unsigned int cryptlen
, int ret
,
1282 struct crypto_alg
*alg
)
1284 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1285 atomic64_inc(&alg
->stats
.cipher
.err_cnt
);
1287 atomic64_inc(&alg
->stats
.cipher
.encrypt_cnt
);
1288 atomic64_add(cryptlen
, &alg
->stats
.cipher
.encrypt_tlen
);
1290 crypto_alg_put(alg
);
1292 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt
);
1294 void crypto_stats_skcipher_decrypt(unsigned int cryptlen
, int ret
,
1295 struct crypto_alg
*alg
)
1297 if (ret
&& ret
!= -EINPROGRESS
&& ret
!= -EBUSY
) {
1298 atomic64_inc(&alg
->stats
.cipher
.err_cnt
);
1300 atomic64_inc(&alg
->stats
.cipher
.decrypt_cnt
);
1301 atomic64_add(cryptlen
, &alg
->stats
.cipher
.decrypt_tlen
);
1303 crypto_alg_put(alg
);
1305 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt
);
1308 static int __init
crypto_algapi_init(void)
1314 static void __exit
crypto_algapi_exit(void)
1319 module_init(crypto_algapi_init
);
1320 module_exit(crypto_algapi_exit
);
1322 MODULE_LICENSE("GPL");
1323 MODULE_DESCRIPTION("Cryptographic algorithms API");