s390/sha3: Support sha3 performance enhancements

On newer machines the SHA3 performance of CPACF instructions KIMD and
KLMD can be enhanced by using additional modifier bits. This allows the
application to omit initializing the ICV, but also affects the internal
processing of the instructions. Performance is mostly gained when
processing short messages.

The new CPACF feature is backwards compatible with older machines, i.e.
the new modifier bits are ignored on older machines. However, to save the
ICV initialization, the application must detect the MSA level and omit
the ICV initialization only if this feature is supported.

Reviewed-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Joerg Schmidbauer <jschmidb@de.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h
index 65ea12fc87a1fd51f4a641b317ea70aaf6657626..2bb22db54c31a28dc11f31071927fde031d43f5b 100644 (file)
--- a/arch/s390/crypto/sha.h
+++ b/arch/s390/crypto/sha.h
@@ -25,6 +25,7 @@ struct s390_sha_ctx {
        u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)];
        u8 buf[SHA_MAX_BLOCK_SIZE];
        int func;               /* KIMD function to use */
+       int first_message_part;
 };
 
 struct shash_desc;

diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c
index e1350e033a32580a40fb85cc241b44b787252e76..5bba972a2646c60b7bd34bbbff7b77ed52a657d4 100644 (file)
--- a/arch/s390/crypto/sha3_256_s390.c
+++ b/arch/s390/crypto/sha3_256_s390.c
@@ -21,9 +21,11 @@ static int sha3_256_init(struct shash_desc *desc)
 {
        struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-       memset(sctx->state, 0, sizeof(sctx->state));
+       if (!test_facility(86)) /* msa 12 */
+               memset(sctx->state, 0, sizeof(sctx->state));
        sctx->count = 0;
        sctx->func = CPACF_KIMD_SHA3_256;
+       sctx->first_message_part = 1;
 
        return 0;
 }
@@ -88,9 +90,11 @@ static int sha3_224_init(struct shash_desc *desc)
 {
        struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-       memset(sctx->state, 0, sizeof(sctx->state));
+       if (!test_facility(86)) /* msa 12 */
+               memset(sctx->state, 0, sizeof(sctx->state));
        sctx->count = 0;
        sctx->func = CPACF_KIMD_SHA3_224;
+       sctx->first_message_part = 1;
 
        return 0;
 }

diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c
index 06c142ed9bb1d32bcb256c418b6c53b7d398af59..3554cbbafe9c83af90761b610a12d3f7a837375b 100644 (file)
--- a/arch/s390/crypto/sha3_512_s390.c
+++ b/arch/s390/crypto/sha3_512_s390.c
@@ -20,9 +20,11 @@ static int sha3_512_init(struct shash_desc *desc)
 {
        struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-       memset(sctx->state, 0, sizeof(sctx->state));
+       if (!test_facility(86)) /* msa 12 */
+               memset(sctx->state, 0, sizeof(sctx->state));
        sctx->count = 0;
        sctx->func = CPACF_KIMD_SHA3_512;
+       sctx->first_message_part = 1;
 
        return 0;
 }
@@ -97,9 +99,11 @@ static int sha3_384_init(struct shash_desc *desc)
 {
        struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-       memset(sctx->state, 0, sizeof(sctx->state));
+       if (!test_facility(86)) /* msa 12 */
+               memset(sctx->state, 0, sizeof(sctx->state));
        sctx->count = 0;
        sctx->func = CPACF_KIMD_SHA3_384;
+       sctx->first_message_part = 1;
 
        return 0;
 }

diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c
index 686fe7aa192f457b9862c9a76298cea85acc128d..f6c7fda21abc82624340bdd95027b1769a0a70e8 100644 (file)
--- a/arch/s390/crypto/sha_common.c
+++ b/arch/s390/crypto/sha_common.c
@@ -18,6 +18,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len)
        struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
        unsigned int bsize = crypto_shash_blocksize(desc->tfm);
        unsigned int index, n;
+       int fc;
 
        /* how much is already in the buffer? */
        index = ctx->count % bsize;
@@ -26,10 +27,15 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len)
        if ((index + len) < bsize)
                goto store;
 
+       fc = ctx->func;
+       if (ctx->first_message_part)
+               fc |= test_facility(86) ? CPACF_KIMD_NIP : 0;
+
        /* process one stored block */
        if (index) {
                memcpy(ctx->buf + index, data, bsize - index);
-               cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize);
+               cpacf_kimd(fc, ctx->state, ctx->buf, bsize);
+               ctx->first_message_part = 0;
                data += bsize - index;
                len -= bsize - index;
                index = 0;
@@ -38,7 +44,8 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len)
        /* process as many blocks as possible */
        if (len >= bsize) {
                n = (len / bsize) * bsize;
-               cpacf_kimd(ctx->func, ctx->state, data, n);
+               cpacf_kimd(fc, ctx->state, data, n);
+               ctx->first_message_part = 0;
                data += n;
                len -= n;
        }
@@ -75,7 +82,7 @@ int s390_sha_final(struct shash_desc *desc, u8 *out)
        unsigned int bsize = crypto_shash_blocksize(desc->tfm);
        u64 bits;
        unsigned int n;
-       int mbl_offset;
+       int mbl_offset, fc;
 
        n = ctx->count % bsize;
        bits = ctx->count * 8;
@@ -109,7 +116,11 @@ int s390_sha_final(struct shash_desc *desc, u8 *out)
                return -EINVAL;
        }
 
-       cpacf_klmd(ctx->func, ctx->state, ctx->buf, n);
+       fc = ctx->func;
+       fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0;
+       if (ctx->first_message_part)
+               fc |= CPACF_KLMD_NIP;
+       cpacf_klmd(fc, ctx->state, ctx->buf, n);
 
        /* copy digest to out */
        memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm));

diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h
index 2dbe2e2088bb32cbce21f36b959b9e4fc762c6e0..4b6a8c26e1cee6c1af35878220904d5fdee0c033 100644 (file)
--- a/arch/s390/include/asm/cpacf.h
+++ b/arch/s390/include/asm/cpacf.h
@@ -171,6 +171,14 @@
 #define CPACF_KMA_LAAD 0x200   /* Last-AAD */
 #define CPACF_KMA_HS   0x400   /* Hash-subkey Supplied */
 
+/*
+ * Flags for the KIMD/KLMD (COMPUTE INTERMEDIATE/LAST MESSAGE DIGEST)
+ * instructions
+ */
+#define CPACF_KIMD_NIP         0x8000
+#define CPACF_KLMD_DUFOP       0x4000
+#define CPACF_KLMD_NIP         0x8000
+
 typedef struct { unsigned char bytes[16]; } cpacf_mask_t;
 
 /*
@@ -397,7 +405,7 @@ static inline void cpacf_kimd(unsigned long func, void *param,
        asm volatile(
                "       lgr     0,%[fc]\n"
                "       lgr     1,%[pba]\n"
-               "0:     .insn   rre,%[opc] << 16,0,%[src]\n"
+               "0:     .insn   rrf,%[opc] << 16,0,%[src],8,0\n"
                "       brc     1,0b\n" /* handle partial completion */
                : [src] "+&d" (s.pair)
                : [fc] "d" (func), [pba] "d" ((unsigned long)(param)),
@@ -422,7 +430,7 @@ static inline void cpacf_klmd(unsigned long func, void *param,
        asm volatile(
                "       lgr     0,%[fc]\n"
                "       lgr     1,%[pba]\n"
-               "0:     .insn   rre,%[opc] << 16,0,%[src]\n"
+               "0:     .insn   rrf,%[opc] << 16,0,%[src],8,0\n"
                "       brc     1,0b\n" /* handle partial completion */
                : [src] "+&d" (s.pair)
                : [fc] "d" (func), [pba] "d" ((unsigned long)param),