crypto: arm64/poly1305 - Add block-only interface

Add block-only interface.

Also remove the unnecessary SIMD fallback path.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/arm64/lib/crypto/Makefile b/arch/arm64/lib/crypto/Makefile
index ac624c3effdaf47ebbb53915191dca81372af606..6207088397a73e3f3960eac08ab82465f72cd1a2 100644 (file)
--- a/arch/arm64/lib/crypto/Makefile
+++ b/arch/arm64/lib/crypto/Makefile
@@ -5,7 +5,8 @@ chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o
 
 obj-$(CONFIG_CRYPTO_POLY1305_NEON) += poly1305-neon.o
 poly1305-neon-y := poly1305-core.o poly1305-glue.o
-AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_init_arm64
+AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch
+AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch
 
 quiet_cmd_perlasm = PERLASM $@
       cmd_perlasm = $(PERL) $(<) void $(@)

diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c
index 906970dd5373234f3b53cd597c66c5df09961c9d..d66a820e32d57faf92828f61fb83c38353ce7699 100644 (file)
--- a/arch/arm64/lib/crypto/poly1305-glue.c
+++ b/arch/arm64/lib/crypto/poly1305-glue.c
@@ -7,32 +7,60 @@
 
 #include <asm/hwcap.h>
 #include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/poly1305.h>
-#include <crypto/internal/simd.h>
+#include <crypto/internal/poly1305.h>
 #include <linux/cpufeature.h>
 #include <linux/jump_label.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
 
-asmlinkage void poly1305_init_arm64(void *state, const u8 *key);
-asmlinkage void poly1305_blocks(void *state, const u8 *src, u32 len, u32 hibit);
-asmlinkage void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit);
-asmlinkage void poly1305_emit(void *state, u8 *digest, const u32 *nonce);
+asmlinkage void poly1305_block_init_arch(
+       struct poly1305_block_state *state,
+       const u8 raw_key[POLY1305_BLOCK_SIZE]);
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
+asmlinkage void poly1305_blocks(struct poly1305_block_state *state,
+                               const u8 *src, u32 len, u32 hibit);
+asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
+                                    const u8 *src, u32 len, u32 hibit);
+asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
+                                  u8 digest[POLY1305_DIGEST_SIZE],
+                                  const u32 nonce[4]);
+EXPORT_SYMBOL_GPL(poly1305_emit_arch);
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 
 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
 {
-       poly1305_init_arm64(&dctx->h, key);
        dctx->s[0] = get_unaligned_le32(key + 16);
        dctx->s[1] = get_unaligned_le32(key + 20);
        dctx->s[2] = get_unaligned_le32(key + 24);
        dctx->s[3] = get_unaligned_le32(key + 28);
        dctx->buflen = 0;
+       poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);
 
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
+                         unsigned int len, u32 padbit)
+{
+       len = round_down(len, POLY1305_BLOCK_SIZE);
+       if (static_branch_likely(&have_neon)) {
+               do {
+                       unsigned int todo = min_t(unsigned int, len, SZ_4K);
+
+                       kernel_neon_begin();
+                       poly1305_blocks_neon(state, src, todo, 1);
+                       kernel_neon_end();
+
+                       len -= todo;
+                       src += todo;
+               } while (len);
+       } else
+               poly1305_blocks(state, src, len, 1);
+}
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
+
 void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
                          unsigned int nbytes)
 {
@@ -45,29 +73,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
                dctx->buflen += bytes;
 
                if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-                       poly1305_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
+                       poly1305_blocks_arch(&dctx->state, dctx->buf,
+                                            POLY1305_BLOCK_SIZE, 1);
                        dctx->buflen = 0;
                }
        }
 
        if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
-               unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
-
-               if (static_branch_likely(&have_neon) && crypto_simd_usable()) {
-                       do {
-                               unsigned int todo = min_t(unsigned int, len, SZ_4K);
-
-                               kernel_neon_begin();
-                               poly1305_blocks_neon(&dctx->h, src, todo, 1);
-                               kernel_neon_end();
-
-                               len -= todo;
-                               src += todo;
-                       } while (len);
-               } else {
-                       poly1305_blocks(&dctx->h, src, len, 1);
-                       src += len;
-               }
+               poly1305_blocks_arch(&dctx->state, src, nbytes, 1);
+               src += round_down(nbytes, POLY1305_BLOCK_SIZE);
                nbytes %= POLY1305_BLOCK_SIZE;
        }
 
@@ -84,10 +98,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
                dctx->buf[dctx->buflen++] = 1;
                memset(dctx->buf + dctx->buflen, 0,
                       POLY1305_BLOCK_SIZE - dctx->buflen);
-               poly1305_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
+               poly1305_blocks_arch(&dctx->state, dctx->buf,
+                                    POLY1305_BLOCK_SIZE, 0);
        }
 
-       poly1305_emit(&dctx->h, dst, dctx->s);
+       poly1305_emit_arch(&dctx->h, dst, dctx->s);
        memzero_explicit(dctx, sizeof(*dctx));
 }
 EXPORT_SYMBOL(poly1305_final_arch);