crypto: talitos - fix SEC1 32k ahash request limitation

Since commit c662b043cdca ("crypto: af_alg/hash: Support
MSG_SPLICE_PAGES"), the crypto core may pass large scatterlists spanning
multiple pages to drivers supporting ahash operations. As a result, a
driver can now receive large ahash requests.

The SEC1 engine has a limitation where a single descriptor cannot
process more than 32k of data. The current implementation attempts to
handle the entire request within a single descriptor, which leads to
failures raised by the driver:

  "length exceeds h/w max limit"

Address this limitation by splitting large ahash requests into multiple
descriptors, each respecting the 32k hardware limit. This allows
processing arbitrarily large requests.

Cc: stable@vger.kernel.org
Fixes: c662b043cdca ("crypto: af_alg/hash: Support MSG_SPLICE_PAGES")
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index e8c0db687c57fdd6ada21541a30263e0eb795f76..4c325fa0eac193c684ff41ab3004815e2bf8c8e1 100644 (file)
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -12,6 +12,7 @@
  * All rights reserved.
  */
 
+#include <linux/workqueue.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
@@ -870,10 +871,18 @@ struct talitos_ahash_req_ctx {
        unsigned int swinit;
        unsigned int first;
        unsigned int last;
+       unsigned int last_request;
        unsigned int to_hash_later;
        unsigned int nbuf;
        struct scatterlist bufsl[2];
        struct scatterlist *psrc;
+
+       struct scatterlist request_bufsl[2];
+       struct ahash_request *areq;
+       struct scatterlist *request_sl;
+       unsigned int remaining_ahash_request_bytes;
+       unsigned int current_ahash_request_bytes;
+       struct work_struct sec1_ahash_process_remaining;
 };
 
 struct talitos_export_state {
@@ -1759,7 +1768,20 @@ static void ahash_done(struct device *dev,
 
        kfree(edesc);
 
-       ahash_request_complete(areq, err);
+       if (err) {
+               ahash_request_complete(areq, err);
+               return;
+       }
+
+       req_ctx->remaining_ahash_request_bytes -=
+               req_ctx->current_ahash_request_bytes;
+
+       if (!req_ctx->remaining_ahash_request_bytes) {
+               ahash_request_complete(areq, 0);
+               return;
+       }
+
+       schedule_work(&req_ctx->sec1_ahash_process_remaining);
 }
 
 /*
@@ -1925,60 +1947,7 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
                                   nbytes, 0, 0, 0, areq->base.flags, false);
 }
 
-static int ahash_init(struct ahash_request *areq)
-{
-       struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
-       struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
-       struct device *dev = ctx->dev;
-       struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
-       unsigned int size;
-       dma_addr_t dma;
-
-       /* Initialize the context */
-       req_ctx->buf_idx = 0;
-       req_ctx->nbuf = 0;
-       req_ctx->first = 1; /* first indicates h/w must init its context */
-       req_ctx->swinit = 0; /* assume h/w init of context */
-       size =  (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
-                       ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
-                       : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
-       req_ctx->hw_context_size = size;
-
-       dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
-                            DMA_TO_DEVICE);
-       dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
-
-       return 0;
-}
-
-/*
- * on h/w without explicit sha224 support, we initialize h/w context
- * manually with sha224 constants, and tell it to run sha256.
- */
-static int ahash_init_sha224_swinit(struct ahash_request *areq)
-{
-       struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
-
-       req_ctx->hw_context[0] = SHA224_H0;
-       req_ctx->hw_context[1] = SHA224_H1;
-       req_ctx->hw_context[2] = SHA224_H2;
-       req_ctx->hw_context[3] = SHA224_H3;
-       req_ctx->hw_context[4] = SHA224_H4;
-       req_ctx->hw_context[5] = SHA224_H5;
-       req_ctx->hw_context[6] = SHA224_H6;
-       req_ctx->hw_context[7] = SHA224_H7;
-
-       /* init 64-bit count */
-       req_ctx->hw_context[8] = 0;
-       req_ctx->hw_context[9] = 0;
-
-       ahash_init(areq);
-       req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
-
-       return 0;
-}
-
-static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
+static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes)
 {
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
        struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
@@ -1997,12 +1966,12 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
 
        if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
                /* Buffer up to one whole block */
-               nents = sg_nents_for_len(areq->src, nbytes);
+               nents = sg_nents_for_len(req_ctx->request_sl, nbytes);
                if (nents < 0) {
                        dev_err(dev, "Invalid number of src SG.\n");
                        return nents;
                }
-               sg_copy_to_buffer(areq->src, nents,
+               sg_copy_to_buffer(req_ctx->request_sl, nents,
                                  ctx_buf + req_ctx->nbuf, nbytes);
                req_ctx->nbuf += nbytes;
                return 0;
@@ -2029,7 +1998,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
                sg_init_table(req_ctx->bufsl, nsg);
                sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
                if (nsg > 1)
-                       sg_chain(req_ctx->bufsl, 2, areq->src);
+                       sg_chain(req_ctx->bufsl, 2, req_ctx->request_sl);
                req_ctx->psrc = req_ctx->bufsl;
        } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
                int offset;
@@ -2038,26 +2007,26 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
                        offset = blocksize - req_ctx->nbuf;
                else
                        offset = nbytes_to_hash - req_ctx->nbuf;
-               nents = sg_nents_for_len(areq->src, offset);
+               nents = sg_nents_for_len(req_ctx->request_sl, offset);
                if (nents < 0) {
                        dev_err(dev, "Invalid number of src SG.\n");
                        return nents;
                }
-               sg_copy_to_buffer(areq->src, nents,
+               sg_copy_to_buffer(req_ctx->request_sl, nents,
                                  ctx_buf + req_ctx->nbuf, offset);
                req_ctx->nbuf += offset;
-               req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src,
+               req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, req_ctx->request_sl,
                                                 offset);
        } else
-               req_ctx->psrc = areq->src;
+               req_ctx->psrc = req_ctx->request_sl;
 
        if (to_hash_later) {
-               nents = sg_nents_for_len(areq->src, nbytes);
+               nents = sg_nents_for_len(req_ctx->request_sl, nbytes);
                if (nents < 0) {
                        dev_err(dev, "Invalid number of src SG.\n");
                        return nents;
                }
-               sg_pcopy_to_buffer(areq->src, nents,
+               sg_pcopy_to_buffer(req_ctx->request_sl, nents,
                                   req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
                                      to_hash_later,
                                      nbytes - to_hash_later);
@@ -2065,7 +2034,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
        req_ctx->to_hash_later = to_hash_later;
 
        /* Allocate extended descriptor */
-       edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
+       edesc = ahash_edesc_alloc(req_ctx->areq, nbytes_to_hash);
        if (IS_ERR(edesc))
                return PTR_ERR(edesc);
 
@@ -2087,14 +2056,123 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
        if (ctx->keylen && (req_ctx->first || req_ctx->last))
                edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
 
-       return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);
+       return common_nonsnoop_hash(edesc, req_ctx->areq, nbytes_to_hash, ahash_done);
 }
 
-static int ahash_update(struct ahash_request *areq)
+static void sec1_ahash_process_remaining(struct work_struct *work)
+{
+       struct talitos_ahash_req_ctx *req_ctx =
+               container_of(work, struct talitos_ahash_req_ctx,
+                            sec1_ahash_process_remaining);
+       int err = 0;
+
+       req_ctx->request_sl = scatterwalk_ffwd(req_ctx->request_bufsl,
+                                              req_ctx->request_sl, TALITOS1_MAX_DATA_LEN);
+
+       if (req_ctx->remaining_ahash_request_bytes > TALITOS1_MAX_DATA_LEN)
+               req_ctx->current_ahash_request_bytes = TALITOS1_MAX_DATA_LEN;
+       else {
+               req_ctx->current_ahash_request_bytes =
+                       req_ctx->remaining_ahash_request_bytes;
+
+               if (req_ctx->last_request)
+                       req_ctx->last = 1;
+       }
+
+       err = ahash_process_req_one(req_ctx->areq,
+                                   req_ctx->current_ahash_request_bytes);
+
+       if (err != -EINPROGRESS)
+               ahash_request_complete(req_ctx->areq, err);
+}
+
+static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
+{
+       struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+       struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+       struct device *dev = ctx->dev;
+       struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+       struct talitos_private *priv = dev_get_drvdata(dev);
+       bool is_sec1 = has_ftr_sec1(priv);
+
+       req_ctx->areq = areq;
+       req_ctx->request_sl = areq->src;
+       req_ctx->remaining_ahash_request_bytes = nbytes;
+
+       if (is_sec1) {
+               if (nbytes > TALITOS1_MAX_DATA_LEN)
+                       nbytes = TALITOS1_MAX_DATA_LEN;
+               else if (req_ctx->last_request)
+                       req_ctx->last = 1;
+       }
+
+       req_ctx->current_ahash_request_bytes = nbytes;
+
+       return ahash_process_req_one(req_ctx->areq,
+                                    req_ctx->current_ahash_request_bytes);
+}
+
+static int ahash_init(struct ahash_request *areq)
 {
+       struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+       struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+       struct device *dev = ctx->dev;
        struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+       unsigned int size;
+       dma_addr_t dma;
 
+       /* Initialize the context */
+       req_ctx->buf_idx = 0;
+       req_ctx->nbuf = 0;
+       req_ctx->first = 1; /* first indicates h/w must init its context */
+       req_ctx->swinit = 0; /* assume h/w init of context */
+       size =  (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
+                       ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
+                       : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
+       req_ctx->hw_context_size = size;
+       req_ctx->last_request = 0;
        req_ctx->last = 0;
+       INIT_WORK(&req_ctx->sec1_ahash_process_remaining, sec1_ahash_process_remaining);
+
+       dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
+                            DMA_TO_DEVICE);
+       dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
+
+       return 0;
+}
+
+/*
+ * on h/w without explicit sha224 support, we initialize h/w context
+ * manually with sha224 constants, and tell it to run sha256.
+ */
+static int ahash_init_sha224_swinit(struct ahash_request *areq)
+{
+       struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+       req_ctx->hw_context[0] = SHA224_H0;
+       req_ctx->hw_context[1] = SHA224_H1;
+       req_ctx->hw_context[2] = SHA224_H2;
+       req_ctx->hw_context[3] = SHA224_H3;
+       req_ctx->hw_context[4] = SHA224_H4;
+       req_ctx->hw_context[5] = SHA224_H5;
+       req_ctx->hw_context[6] = SHA224_H6;
+       req_ctx->hw_context[7] = SHA224_H7;
+
+       /* init 64-bit count */
+       req_ctx->hw_context[8] = 0;
+       req_ctx->hw_context[9] = 0;
+
+       ahash_init(areq);
+       req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
+
+       return 0;
+}
+
+static int ahash_update(struct ahash_request *areq)
+{
+       struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+       req_ctx->last_request = 0;
 
        return ahash_process_req(areq, areq->nbytes);
 }
@@ -2103,7 +2181,7 @@ static int ahash_final(struct ahash_request *areq)
 {
        struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
 
-       req_ctx->last = 1;
+       req_ctx->last_request = 1;
 
        return ahash_process_req(areq, 0);
 }
@@ -2112,7 +2190,7 @@ static int ahash_finup(struct ahash_request *areq)
 {
        struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
 
-       req_ctx->last = 1;
+       req_ctx->last_request = 1;
 
        return ahash_process_req(areq, areq->nbytes);
 }