lib/crypto: powerpc/aes: Migrate SPE optimized code into library

Move the PowerPC SPE AES assembly code into lib/crypto/, wire the key
expansion and single-block en/decryption functions up to the AES library
API, and remove the superseded "aes-ppc-spe" crypto_cipher algorithm.

The result is that both the AES library and crypto_cipher APIs are now
optimized with SPE, whereas previously only crypto_cipher was (and
optimizations weren't enabled by default, which this commit fixes too).

Note that many of the functions in the PowerPC SPE assembly code are
still used by the AES mode implementations in arch/powerpc/crypto/.  For
now, just export these functions.  These exports will go away once the
AES modes are migrated to the library as well.  (Trying to split up the
assembly files seemed like much more trouble than it would be worth.)

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig
index 662aed46f9c795a28acc0ae19b8249c581eee62d..2d056f1fc90fc6c4a29f0f6dc8ececa3c69b6d59 100644 (file)
--- a/arch/powerpc/crypto/Kconfig
+++ b/arch/powerpc/crypto/Kconfig
@@ -5,9 +5,9 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)"
 config CRYPTO_AES_PPC_SPE
        tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)"
        depends on SPE
+       select CRYPTO_LIB_AES
        select CRYPTO_SKCIPHER
        help
-         Block ciphers: AES cipher algorithms (FIPS-197)
          Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes
 
          Architecture: powerpc using:

diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile
index 5960e5300db71e315c21ca95f79cd8924f74affa..e22310da86b55ff82872d928009bf09b498695a1 100644 (file)
--- a/arch/powerpc/crypto/Makefile
+++ b/arch/powerpc/crypto/Makefile
@@ -9,7 +9,7 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o
 obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o
 obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
 
-aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o
+aes-ppc-spe-y := aes-spe-glue.o
 aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
 vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
 

diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c
index efab78a3a8f6ba8f4643c37c866ec210bfb783e0..7d2827e652408dc5c1b72a7d50a982856c8a6d28 100644 (file)
--- a/arch/powerpc/crypto/aes-spe-glue.c
+++ b/arch/powerpc/crypto/aes-spe-glue.c
@@ -51,30 +51,6 @@ struct ppc_xts_ctx {
        u32 rounds;
 };
 
-extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
-extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
-extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-                           u32 bytes);
-extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-                           u32 bytes);
-extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-                           u32 bytes, u8 *iv);
-extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-                           u32 bytes, u8 *iv);
-extern void ppc_crypt_ctr  (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-                           u32 bytes, u8 *iv);
-extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-                           u32 bytes, u8 *iv, u32 *key_twk);
-extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-                           u32 bytes, u8 *iv, u32 *key_twk);
-
-extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
-extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
-extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
-
-extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
-                                    unsigned int key_len);
-
 static void spe_begin(void)
 {
        /* disable preemption and save users SPE registers if required */
@@ -89,10 +65,10 @@ static void spe_end(void)
        preempt_enable();
 }
 
-static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
-               unsigned int key_len)
+static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
+                                  const u8 *in_key, unsigned int key_len)
 {
-       struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 
        switch (key_len) {
        case AES_KEYSIZE_128:
@@ -116,12 +92,6 @@ static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
        return 0;
 }
 
-static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
-                                  const u8 *in_key, unsigned int key_len)
-{
-       return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
-}
-
 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
                   unsigned int key_len)
 {
@@ -159,24 +129,6 @@ static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
        return 0;
 }
 
-static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-       struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-       spe_begin();
-       ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
-       spe_end();
-}
-
-static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-       struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-       spe_begin();
-       ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
-       spe_end();
-}
-
 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -410,26 +362,6 @@ static int ppc_xts_decrypt(struct skcipher_request *req)
  * with kmalloc() in the crypto infrastructure
  */
 
-static struct crypto_alg aes_cipher_alg = {
-       .cra_name               =       "aes",
-       .cra_driver_name        =       "aes-ppc-spe",
-       .cra_priority           =       300,
-       .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
-       .cra_blocksize          =       AES_BLOCK_SIZE,
-       .cra_ctxsize            =       sizeof(struct ppc_aes_ctx),
-       .cra_alignmask          =       0,
-       .cra_module             =       THIS_MODULE,
-       .cra_u                  =       {
-               .cipher = {
-                       .cia_min_keysize        =       AES_MIN_KEY_SIZE,
-                       .cia_max_keysize        =       AES_MAX_KEY_SIZE,
-                       .cia_setkey             =       ppc_aes_setkey,
-                       .cia_encrypt            =       ppc_aes_encrypt,
-                       .cia_decrypt            =       ppc_aes_decrypt
-               }
-       }
-};
-
 static struct skcipher_alg aes_skcipher_algs[] = {
        {
                .base.cra_name          =       "ecb(aes)",
@@ -488,22 +420,12 @@ static struct skcipher_alg aes_skcipher_algs[] = {
 
 static int __init ppc_aes_mod_init(void)
 {
-       int err;
-
-       err = crypto_register_alg(&aes_cipher_alg);
-       if (err)
-               return err;
-
-       err = crypto_register_skciphers(aes_skcipher_algs,
-                                       ARRAY_SIZE(aes_skcipher_algs));
-       if (err)
-               crypto_unregister_alg(&aes_cipher_alg);
-       return err;
+       return crypto_register_skciphers(aes_skcipher_algs,
+                                        ARRAY_SIZE(aes_skcipher_algs));
 }
 
 static void __exit ppc_aes_mod_fini(void)
 {
-       crypto_unregister_alg(&aes_cipher_alg);
        crypto_unregister_skciphers(aes_skcipher_algs,
                                    ARRAY_SIZE(aes_skcipher_algs));
 }

diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index 18af1acbde58bf8178b1eb67af78bf661c05fef0..c893c9214cb7e5ed18ba6cf423c11a85881f922e 100644 (file)
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -20,10 +20,22 @@
 
 union aes_enckey_arch {
        u32 rndkeys[AES_MAX_KEYLENGTH_U32];
+#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
+#if defined(CONFIG_PPC) && defined(CONFIG_SPE)
+       /* Used unconditionally (when SPE AES code is enabled in kconfig) */
+       u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8);
+#endif
+#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
 };
 
 union aes_invkey_arch {
        u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
+#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
+#if defined(CONFIG_PPC) && defined(CONFIG_SPE)
+       /* Used unconditionally (when SPE AES code is enabled in kconfig) */
+       u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8);
+#endif
+#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
 };
 
 /**
@@ -124,6 +136,25 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 #ifdef CONFIG_ARM64
 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
                     unsigned int key_len);
+#elif defined(CONFIG_PPC)
+void ppc_expand_key_128(u32 *key_enc, const u8 *key);
+void ppc_expand_key_192(u32 *key_enc, const u8 *key);
+void ppc_expand_key_256(u32 *key_enc, const u8 *key);
+void ppc_generate_decrypt_key(u32 *key_dec, u32 *key_enc, unsigned int key_len);
+void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
+                    u32 bytes);
+void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
+                    u32 bytes);
+void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,
+                    u8 *iv);
+void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,
+                    u8 *iv);
+void ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,
+                  u8 *iv);
+void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,
+                    u8 *iv, u32 *key_twk);
+void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,
+                    u8 *iv, u32 *key_twk);
 #endif
 
 /**

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index ead47b2a7db68b3c6121192b87c1ee81dff05d9a..cfc6171203d02ba70ce564c547928c234dd020a0 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -16,6 +16,7 @@ config CRYPTO_LIB_AES_ARCH
        depends on CRYPTO_LIB_AES && !UML && !KMSAN
        default y if ARM
        default y if ARM64
+       default y if PPC && SPE
 
 config CRYPTO_LIB_AESCFB
        tristate

diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 1b690c63fafb5f49f60de54104398ec9669298fd..d68fde00410488bec20202d4acbcc714887ed3ba 100644 (file)
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -29,6 +29,15 @@ libaes-y += arm64/aes-cipher-core.o
 libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o
 endif
 
+ifeq ($(CONFIG_PPC),y)
+ifeq ($(CONFIG_SPE),y)
+libaes-y += powerpc/aes-spe-core.o \
+           powerpc/aes-spe-keys.o \
+           powerpc/aes-spe-modes.o \
+           powerpc/aes-tab-4k.o
+endif
+endif # CONFIG_PPC
+
 endif # CONFIG_CRYPTO_LIB_AES_ARCH
 
 ################################################################################

diff --git a/arch/powerpc/crypto/aes-spe-core.S b/lib/crypto/powerpc/aes-spe-core.S
similarity index 100%
rename from arch/powerpc/crypto/aes-spe-core.S
rename to lib/crypto/powerpc/aes-spe-core.S

diff --git a/arch/powerpc/crypto/aes-spe-keys.S b/lib/crypto/powerpc/aes-spe-keys.S
similarity index 100%
rename from arch/powerpc/crypto/aes-spe-keys.S
rename to lib/crypto/powerpc/aes-spe-keys.S

diff --git a/arch/powerpc/crypto/aes-spe-modes.S b/lib/crypto/powerpc/aes-spe-modes.S
similarity index 100%
rename from arch/powerpc/crypto/aes-spe-modes.S
rename to lib/crypto/powerpc/aes-spe-modes.S

diff --git a/arch/powerpc/crypto/aes-spe-regs.h b/lib/crypto/powerpc/aes-spe-regs.h
similarity index 100%
rename from arch/powerpc/crypto/aes-spe-regs.h
rename to lib/crypto/powerpc/aes-spe-regs.h

diff --git a/arch/powerpc/crypto/aes-tab-4k.S b/lib/crypto/powerpc/aes-tab-4k.S
similarity index 100%
rename from arch/powerpc/crypto/aes-tab-4k.S
rename to lib/crypto/powerpc/aes-tab-4k.S

diff --git a/lib/crypto/powerpc/aes.h b/lib/crypto/powerpc/aes.h
new file mode 100644 (file)
index 0000000..cf22020
--- /dev/null
+++ b/lib/crypto/powerpc/aes.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ * Copyright 2026 Google LLC
+ */
+#include <asm/simd.h>
+#include <asm/switch_to.h>
+#include <linux/cpufeature.h>
+#include <linux/jump_label.h>
+#include <linux/preempt.h>
+#include <linux/uaccess.h>
+
+EXPORT_SYMBOL_GPL(ppc_expand_key_128);
+EXPORT_SYMBOL_GPL(ppc_expand_key_192);
+EXPORT_SYMBOL_GPL(ppc_expand_key_256);
+EXPORT_SYMBOL_GPL(ppc_generate_decrypt_key);
+EXPORT_SYMBOL_GPL(ppc_encrypt_ecb);
+EXPORT_SYMBOL_GPL(ppc_decrypt_ecb);
+EXPORT_SYMBOL_GPL(ppc_encrypt_cbc);
+EXPORT_SYMBOL_GPL(ppc_decrypt_cbc);
+EXPORT_SYMBOL_GPL(ppc_crypt_ctr);
+EXPORT_SYMBOL_GPL(ppc_encrypt_xts);
+EXPORT_SYMBOL_GPL(ppc_decrypt_xts);
+
+void ppc_encrypt_aes(u8 *out, const u8 *in, const u32 *key_enc, u32 rounds);
+void ppc_decrypt_aes(u8 *out, const u8 *in, const u32 *key_dec, u32 rounds);
+
+static void spe_begin(void)
+{
+       /* disable preemption and save users SPE registers if required */
+       preempt_disable();
+       enable_kernel_spe();
+}
+
+static void spe_end(void)
+{
+       disable_kernel_spe();
+       /* reenable preemption */
+       preempt_enable();
+}
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+                               union aes_invkey_arch *inv_k,
+                               const u8 *in_key, int key_len, int nrounds)
+{
+       if (key_len == AES_KEYSIZE_128)
+               ppc_expand_key_128(k->spe_enc_key, in_key);
+       else if (key_len == AES_KEYSIZE_192)
+               ppc_expand_key_192(k->spe_enc_key, in_key);
+       else
+               ppc_expand_key_256(k->spe_enc_key, in_key);
+
+       if (inv_k)
+               ppc_generate_decrypt_key(inv_k->spe_dec_key, k->spe_enc_key,
+                                        key_len);
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+                            u8 out[AES_BLOCK_SIZE],
+                            const u8 in[AES_BLOCK_SIZE])
+{
+       spe_begin();
+       ppc_encrypt_aes(out, in, key->k.spe_enc_key, key->nrounds / 2 - 1);
+       spe_end();
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+                            u8 out[AES_BLOCK_SIZE],
+                            const u8 in[AES_BLOCK_SIZE])
+{
+       spe_begin();
+       ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1);
+       spe_end();
+}