From 0892c91b81cc889c95dc03b095b9f4a6fdf93106 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 12 Jan 2026 11:20:10 -0800 Subject: [PATCH] lib/crypto: powerpc/aes: Migrate SPE optimized code into library Move the PowerPC SPE AES assembly code into lib/crypto/, wire the key expansion and single-block en/decryption functions up to the AES library API, and remove the superseded "aes-ppc-spe" crypto_cipher algorithm. The result is that both the AES library and crypto_cipher APIs are now optimized with SPE, whereas previously only crypto_cipher was (and optimizations weren't enabled by default, which this commit fixes too). Note that many of the functions in the PowerPC SPE assembly code are still used by the AES mode implementations in arch/powerpc/crypto/. For now, just export these functions. These exports will go away once the AES modes are migrated to the library as well. (Trying to split up the assembly files seemed like much more trouble than it would be worth.) Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260112192035.10427-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/powerpc/crypto/Kconfig | 2 +- arch/powerpc/crypto/Makefile | 2 +- arch/powerpc/crypto/aes-spe-glue.c | 88 ++----------------- include/crypto/aes.h | 31 +++++++ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 9 ++ .../crypto/powerpc}/aes-spe-core.S | 0 .../crypto/powerpc}/aes-spe-keys.S | 0 .../crypto/powerpc}/aes-spe-modes.S | 0 .../crypto/powerpc}/aes-spe-regs.h | 0 .../crypto/powerpc}/aes-tab-4k.S | 0 lib/crypto/powerpc/aes.h | 74 ++++++++++++++++ 12 files changed, 122 insertions(+), 85 deletions(-) rename {arch/powerpc/crypto => lib/crypto/powerpc}/aes-spe-core.S (100%) rename {arch/powerpc/crypto => lib/crypto/powerpc}/aes-spe-keys.S (100%) rename {arch/powerpc/crypto => lib/crypto/powerpc}/aes-spe-modes.S (100%) rename {arch/powerpc/crypto => lib/crypto/powerpc}/aes-spe-regs.h (100%) rename {arch/powerpc/crypto => lib/crypto/powerpc}/aes-tab-4k.S (100%) create mode 100644 lib/crypto/powerpc/aes.h diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 662aed46f9c79..2d056f1fc90fc 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -5,9 +5,9 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" config CRYPTO_AES_PPC_SPE tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" depends on SPE + select CRYPTO_LIB_AES select CRYPTO_SKCIPHER help - Block ciphers: AES cipher algorithms (FIPS-197) Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes Architecture: powerpc using: diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 5960e5300db71..e22310da86b55 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -9,7 +9,7 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o -aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o +aes-ppc-spe-y := aes-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c index efab78a3a8f6b..7d2827e652408 100644 --- a/arch/powerpc/crypto/aes-spe-glue.c +++ b/arch/powerpc/crypto/aes-spe-glue.c @@ -51,30 +51,6 @@ struct ppc_xts_ctx { u32 rounds; }; -extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds); -extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds); -extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, - u32 bytes); -extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, - u32 bytes); -extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, - u32 bytes, u8 *iv); -extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, - u32 bytes, u8 *iv); -extern void ppc_crypt_ctr (u8 *out, const u8 *in, u32 *key_enc, u32 rounds, - u32 bytes, u8 *iv); -extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, - u32 bytes, u8 *iv, u32 *key_twk); -extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, - u32 bytes, u8 *iv, u32 *key_twk); - -extern void ppc_expand_key_128(u32 *key_enc, const u8 *key); -extern void ppc_expand_key_192(u32 *key_enc, const u8 *key); -extern void ppc_expand_key_256(u32 *key_enc, const u8 *key); - -extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc, - unsigned int key_len); - static void spe_begin(void) { /* disable preemption and save users SPE registers if required */ @@ -89,10 +65,10 @@ static void spe_end(void) preempt_enable(); } -static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) +static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm, + const u8 *in_key, unsigned int key_len) { - struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); + struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); switch (key_len) { case AES_KEYSIZE_128: @@ -116,12 +92,6 @@ static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, return 0; } -static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm, - const u8 *in_key, unsigned int key_len) -{ - return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len); -} - static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -159,24 +129,6 @@ static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, return 0; } -static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - spe_begin(); - ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds); - spe_end(); -} - -static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - spe_begin(); - ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds); - spe_end(); -} - static int ppc_ecb_crypt(struct skcipher_request *req, bool enc) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -410,26 +362,6 @@ static int ppc_xts_decrypt(struct skcipher_request *req) * with kmalloc() in the crypto infrastructure */ -static struct crypto_alg aes_cipher_alg = { - .cra_name = "aes", - .cra_driver_name = "aes-ppc-spe", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ppc_aes_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = ppc_aes_setkey, - .cia_encrypt = ppc_aes_encrypt, - .cia_decrypt = ppc_aes_decrypt - } - } -}; - static struct skcipher_alg aes_skcipher_algs[] = { { .base.cra_name = "ecb(aes)", @@ -488,22 +420,12 @@ static struct skcipher_alg aes_skcipher_algs[] = { static int __init ppc_aes_mod_init(void) { - int err; - - err = crypto_register_alg(&aes_cipher_alg); - if (err) - return err; - - err = crypto_register_skciphers(aes_skcipher_algs, - ARRAY_SIZE(aes_skcipher_algs)); - if (err) - crypto_unregister_alg(&aes_cipher_alg); - return err; + return crypto_register_skciphers(aes_skcipher_algs, + ARRAY_SIZE(aes_skcipher_algs)); } static void __exit ppc_aes_mod_fini(void) { - crypto_unregister_alg(&aes_cipher_alg); crypto_unregister_skciphers(aes_skcipher_algs, ARRAY_SIZE(aes_skcipher_algs)); } diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 18af1acbde58b..c893c9214cb7e 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -20,10 +20,22 @@ union aes_enckey_arch { u32 rndkeys[AES_MAX_KEYLENGTH_U32]; +#ifdef CONFIG_CRYPTO_LIB_AES_ARCH +#if defined(CONFIG_PPC) && defined(CONFIG_SPE) + /* Used unconditionally (when SPE AES code is enabled in kconfig) */ + u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8); +#endif +#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */ }; union aes_invkey_arch { u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32]; +#ifdef CONFIG_CRYPTO_LIB_AES_ARCH +#if defined(CONFIG_PPC) && defined(CONFIG_SPE) + /* Used unconditionally (when SPE AES code is enabled in kconfig) */ + u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8); +#endif +#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */ }; /** @@ -124,6 +136,25 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, #ifdef CONFIG_ARM64 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); +#elif defined(CONFIG_PPC) +void ppc_expand_key_128(u32 *key_enc, const u8 *key); +void ppc_expand_key_192(u32 *key_enc, const u8 *key); +void ppc_expand_key_256(u32 *key_enc, const u8 *key); +void ppc_generate_decrypt_key(u32 *key_dec, u32 *key_enc, unsigned int key_len); +void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, + u32 bytes); +void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, + u32 bytes); +void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, + u8 *iv); +void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes, + u8 *iv); +void ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, + u8 *iv); +void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, + u8 *iv, u32 *key_twk); +void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes, + u8 *iv, u32 *key_twk); #endif /** diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index ead47b2a7db68..cfc6171203d02 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -16,6 +16,7 @@ config CRYPTO_LIB_AES_ARCH depends on CRYPTO_LIB_AES && !UML && !KMSAN default y if ARM default y if ARM64 + default y if PPC && SPE config CRYPTO_LIB_AESCFB tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 1b690c63fafb5..d68fde0041048 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -29,6 +29,15 @@ libaes-y += arm64/aes-cipher-core.o libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o endif +ifeq ($(CONFIG_PPC),y) +ifeq ($(CONFIG_SPE),y) +libaes-y += powerpc/aes-spe-core.o \ + powerpc/aes-spe-keys.o \ + powerpc/aes-spe-modes.o \ + powerpc/aes-tab-4k.o +endif +endif # CONFIG_PPC + endif # CONFIG_CRYPTO_LIB_AES_ARCH ################################################################################ diff --git a/arch/powerpc/crypto/aes-spe-core.S b/lib/crypto/powerpc/aes-spe-core.S similarity index 100% rename from arch/powerpc/crypto/aes-spe-core.S rename to lib/crypto/powerpc/aes-spe-core.S diff --git a/arch/powerpc/crypto/aes-spe-keys.S b/lib/crypto/powerpc/aes-spe-keys.S similarity index 100% rename from arch/powerpc/crypto/aes-spe-keys.S rename to lib/crypto/powerpc/aes-spe-keys.S diff --git a/arch/powerpc/crypto/aes-spe-modes.S b/lib/crypto/powerpc/aes-spe-modes.S similarity index 100% rename from arch/powerpc/crypto/aes-spe-modes.S rename to lib/crypto/powerpc/aes-spe-modes.S diff --git a/arch/powerpc/crypto/aes-spe-regs.h b/lib/crypto/powerpc/aes-spe-regs.h similarity index 100% rename from arch/powerpc/crypto/aes-spe-regs.h rename to lib/crypto/powerpc/aes-spe-regs.h diff --git a/arch/powerpc/crypto/aes-tab-4k.S b/lib/crypto/powerpc/aes-tab-4k.S similarity index 100% rename from arch/powerpc/crypto/aes-tab-4k.S rename to lib/crypto/powerpc/aes-tab-4k.S diff --git a/lib/crypto/powerpc/aes.h b/lib/crypto/powerpc/aes.h new file mode 100644 index 0000000000000..cf22020f90507 --- /dev/null +++ b/lib/crypto/powerpc/aes.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015 Markus Stockhausen + * Copyright 2026 Google LLC + */ +#include +#include +#include +#include +#include +#include + +EXPORT_SYMBOL_GPL(ppc_expand_key_128); +EXPORT_SYMBOL_GPL(ppc_expand_key_192); +EXPORT_SYMBOL_GPL(ppc_expand_key_256); +EXPORT_SYMBOL_GPL(ppc_generate_decrypt_key); +EXPORT_SYMBOL_GPL(ppc_encrypt_ecb); +EXPORT_SYMBOL_GPL(ppc_decrypt_ecb); +EXPORT_SYMBOL_GPL(ppc_encrypt_cbc); +EXPORT_SYMBOL_GPL(ppc_decrypt_cbc); +EXPORT_SYMBOL_GPL(ppc_crypt_ctr); +EXPORT_SYMBOL_GPL(ppc_encrypt_xts); +EXPORT_SYMBOL_GPL(ppc_decrypt_xts); + +void ppc_encrypt_aes(u8 *out, const u8 *in, const u32 *key_enc, u32 rounds); +void ppc_decrypt_aes(u8 *out, const u8 *in, const u32 *key_dec, u32 rounds); + +static void spe_begin(void) +{ + /* disable preemption and save users SPE registers if required */ + preempt_disable(); + enable_kernel_spe(); +} + +static void spe_end(void) +{ + disable_kernel_spe(); + /* reenable preemption */ + preempt_enable(); +} + +static void aes_preparekey_arch(union aes_enckey_arch *k, + union aes_invkey_arch *inv_k, + const u8 *in_key, int key_len, int nrounds) +{ + if (key_len == AES_KEYSIZE_128) + ppc_expand_key_128(k->spe_enc_key, in_key); + else if (key_len == AES_KEYSIZE_192) + ppc_expand_key_192(k->spe_enc_key, in_key); + else + ppc_expand_key_256(k->spe_enc_key, in_key); + + if (inv_k) + ppc_generate_decrypt_key(inv_k->spe_dec_key, k->spe_enc_key, + key_len); +} + +static void aes_encrypt_arch(const struct aes_enckey *key, + u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE]) +{ + spe_begin(); + ppc_encrypt_aes(out, in, key->k.spe_enc_key, key->nrounds / 2 - 1); + spe_end(); +} + +static void aes_decrypt_arch(const struct aes_key *key, + u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE]) +{ + spe_begin(); + ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1); + spe_end(); +} -- 2.47.3