1 // SPDX-License-Identifier: GPL-2.0
3 * OpenSSL/Cryptogams accelerated Poly1305 transform for arm64
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
11 #include <asm/unaligned.h>
12 #include <crypto/algapi.h>
13 #include <crypto/internal/hash.h>
14 #include <crypto/internal/poly1305.h>
15 #include <crypto/internal/simd.h>
16 #include <linux/cpufeature.h>
17 #include <linux/crypto.h>
18 #include <linux/jump_label.h>
19 #include <linux/module.h>
21 asmlinkage
void poly1305_init_arm64(void *state
, const u8
*key
);
22 asmlinkage
void poly1305_blocks(void *state
, const u8
*src
, u32 len
, u32 hibit
);
23 asmlinkage
void poly1305_blocks_neon(void *state
, const u8
*src
, u32 len
, u32 hibit
);
24 asmlinkage
void poly1305_emit(void *state
, u8
*digest
, const u32
*nonce
);
26 static __ro_after_init
DEFINE_STATIC_KEY_FALSE(have_neon
);
28 void poly1305_init_arch(struct poly1305_desc_ctx
*dctx
, const u8
*key
)
30 poly1305_init_arm64(&dctx
->h
, key
);
31 dctx
->s
[0] = get_unaligned_le32(key
+ 16);
32 dctx
->s
[1] = get_unaligned_le32(key
+ 20);
33 dctx
->s
[2] = get_unaligned_le32(key
+ 24);
34 dctx
->s
[3] = get_unaligned_le32(key
+ 28);
37 EXPORT_SYMBOL(poly1305_init_arch
);
39 static int neon_poly1305_init(struct shash_desc
*desc
)
41 struct poly1305_desc_ctx
*dctx
= shash_desc_ctx(desc
);
50 static void neon_poly1305_blocks(struct poly1305_desc_ctx
*dctx
, const u8
*src
,
51 u32 len
, u32 hibit
, bool do_neon
)
53 if (unlikely(!dctx
->sset
)) {
55 poly1305_init_arch(dctx
, src
);
56 src
+= POLY1305_BLOCK_SIZE
;
57 len
-= POLY1305_BLOCK_SIZE
;
60 if (len
>= POLY1305_BLOCK_SIZE
) {
61 dctx
->s
[0] = get_unaligned_le32(src
+ 0);
62 dctx
->s
[1] = get_unaligned_le32(src
+ 4);
63 dctx
->s
[2] = get_unaligned_le32(src
+ 8);
64 dctx
->s
[3] = get_unaligned_le32(src
+ 12);
65 src
+= POLY1305_BLOCK_SIZE
;
66 len
-= POLY1305_BLOCK_SIZE
;
69 if (len
< POLY1305_BLOCK_SIZE
)
73 len
&= ~(POLY1305_BLOCK_SIZE
- 1);
75 if (static_branch_likely(&have_neon
) && likely(do_neon
))
76 poly1305_blocks_neon(&dctx
->h
, src
, len
, hibit
);
78 poly1305_blocks(&dctx
->h
, src
, len
, hibit
);
81 static void neon_poly1305_do_update(struct poly1305_desc_ctx
*dctx
,
82 const u8
*src
, u32 len
, bool do_neon
)
84 if (unlikely(dctx
->buflen
)) {
85 u32 bytes
= min(len
, POLY1305_BLOCK_SIZE
- dctx
->buflen
);
87 memcpy(dctx
->buf
+ dctx
->buflen
, src
, bytes
);
90 dctx
->buflen
+= bytes
;
92 if (dctx
->buflen
== POLY1305_BLOCK_SIZE
) {
93 neon_poly1305_blocks(dctx
, dctx
->buf
,
94 POLY1305_BLOCK_SIZE
, 1, false);
99 if (likely(len
>= POLY1305_BLOCK_SIZE
)) {
100 neon_poly1305_blocks(dctx
, src
, len
, 1, do_neon
);
101 src
+= round_down(len
, POLY1305_BLOCK_SIZE
);
102 len
%= POLY1305_BLOCK_SIZE
;
107 memcpy(dctx
->buf
, src
, len
);
111 static int neon_poly1305_update(struct shash_desc
*desc
,
112 const u8
*src
, unsigned int srclen
)
114 bool do_neon
= crypto_simd_usable() && srclen
> 128;
115 struct poly1305_desc_ctx
*dctx
= shash_desc_ctx(desc
);
117 if (static_branch_likely(&have_neon
) && do_neon
)
119 neon_poly1305_do_update(dctx
, src
, srclen
, do_neon
);
120 if (static_branch_likely(&have_neon
) && do_neon
)
125 void poly1305_update_arch(struct poly1305_desc_ctx
*dctx
, const u8
*src
,
128 if (unlikely(dctx
->buflen
)) {
129 u32 bytes
= min(nbytes
, POLY1305_BLOCK_SIZE
- dctx
->buflen
);
131 memcpy(dctx
->buf
+ dctx
->buflen
, src
, bytes
);
134 dctx
->buflen
+= bytes
;
136 if (dctx
->buflen
== POLY1305_BLOCK_SIZE
) {
137 poly1305_blocks(&dctx
->h
, dctx
->buf
, POLY1305_BLOCK_SIZE
, 1);
142 if (likely(nbytes
>= POLY1305_BLOCK_SIZE
)) {
143 unsigned int len
= round_down(nbytes
, POLY1305_BLOCK_SIZE
);
145 if (static_branch_likely(&have_neon
) && crypto_simd_usable()) {
147 unsigned int todo
= min_t(unsigned int, len
, SZ_4K
);
150 poly1305_blocks_neon(&dctx
->h
, src
, todo
, 1);
157 poly1305_blocks(&dctx
->h
, src
, len
, 1);
160 nbytes
%= POLY1305_BLOCK_SIZE
;
163 if (unlikely(nbytes
)) {
164 dctx
->buflen
= nbytes
;
165 memcpy(dctx
->buf
, src
, nbytes
);
168 EXPORT_SYMBOL(poly1305_update_arch
);
170 void poly1305_final_arch(struct poly1305_desc_ctx
*dctx
, u8
*dst
)
172 if (unlikely(dctx
->buflen
)) {
173 dctx
->buf
[dctx
->buflen
++] = 1;
174 memset(dctx
->buf
+ dctx
->buflen
, 0,
175 POLY1305_BLOCK_SIZE
- dctx
->buflen
);
176 poly1305_blocks(&dctx
->h
, dctx
->buf
, POLY1305_BLOCK_SIZE
, 0);
179 poly1305_emit(&dctx
->h
, dst
, dctx
->s
);
180 *dctx
= (struct poly1305_desc_ctx
){};
182 EXPORT_SYMBOL(poly1305_final_arch
);
184 static int neon_poly1305_final(struct shash_desc
*desc
, u8
*dst
)
186 struct poly1305_desc_ctx
*dctx
= shash_desc_ctx(desc
);
188 if (unlikely(!dctx
->sset
))
191 poly1305_final_arch(dctx
, dst
);
195 static struct shash_alg neon_poly1305_alg
= {
196 .init
= neon_poly1305_init
,
197 .update
= neon_poly1305_update
,
198 .final
= neon_poly1305_final
,
199 .digestsize
= POLY1305_DIGEST_SIZE
,
200 .descsize
= sizeof(struct poly1305_desc_ctx
),
202 .base
.cra_name
= "poly1305",
203 .base
.cra_driver_name
= "poly1305-neon",
204 .base
.cra_priority
= 200,
205 .base
.cra_blocksize
= POLY1305_BLOCK_SIZE
,
206 .base
.cra_module
= THIS_MODULE
,
209 static int __init
neon_poly1305_mod_init(void)
211 if (!cpu_have_named_feature(ASIMD
))
214 static_branch_enable(&have_neon
);
216 return IS_REACHABLE(CONFIG_CRYPTO_HASH
) ?
217 crypto_register_shash(&neon_poly1305_alg
) : 0;
220 static void __exit
neon_poly1305_mod_exit(void)
222 if (IS_REACHABLE(CONFIG_CRYPTO_HASH
) && cpu_have_named_feature(ASIMD
))
223 crypto_unregister_shash(&neon_poly1305_alg
);
226 module_init(neon_poly1305_mod_init
);
227 module_exit(neon_poly1305_mod_exit
);
229 MODULE_LICENSE("GPL v2");
230 MODULE_ALIAS_CRYPTO("poly1305");
231 MODULE_ALIAS_CRYPTO("poly1305-neon");