From: Paul Louvel Date: Thu, 7 May 2026 14:41:48 +0000 (+0200) Subject: crypto: talitos - add chaining of arbitrary number of descriptor for the SEC1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f126384ed55279c3b676f89d5ab547b8de8df782;p=thirdparty%2Fkernel%2Flinux.git crypto: talitos - add chaining of arbitrary number of descriptor for the SEC1 The SEC1 hardware can process a chain of descriptors without host intervention. Only the hash implementation currently use this feature, but with a chain of at most 2 descriptors added in commit 37b5e8897eb5 ("crypto: talitos - chain in buffered data for ahash on SEC1"). Add supports for chaining an arbitrary number of descriptors in a chain. Adapt the ahash implementation to make it compatible. Cc: stable@vger.kernel.org Signed-off-by: Paul Louvel Signed-off-by: Herbert Xu --- diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index e7ef286f43061..b26b4672a457e 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -273,7 +273,10 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, void *context, int error), void *context) { + struct talitos_edesc *edesc = container_of(desc, struct talitos_edesc, desc); struct talitos_private *priv = dev_get_drvdata(dev); + dma_addr_t dma_desc, prev_dma_desc; + struct talitos_edesc *prev_edesc = NULL; struct talitos_request *request; unsigned long flags; int head; @@ -292,10 +295,31 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, /* map descriptor and save caller data */ if (is_sec1) { - desc->hdr1 = desc->hdr; - request->dma_desc = dma_map_single(dev, &desc->hdr1, + while (edesc) { + edesc->desc.hdr1 = edesc->desc.hdr; + + dma_desc = dma_map_single(dev, &edesc->desc.hdr1, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + if (!prev_edesc) { + request->dma_desc = dma_desc; + goto next; + } + + /* Chain in any previous descriptors. */ + + prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); + + dma_sync_single_for_device(dev, prev_dma_desc, TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); + DMA_TO_DEVICE); + +next: + prev_edesc = edesc; + prev_dma_desc = dma_desc; + edesc = edesc->next_desc; + } } else { request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, @@ -326,6 +350,7 @@ static __be32 get_request_hdr(struct device *dev, struct talitos_request *request, bool is_sec1) { struct talitos_edesc *edesc; + dma_addr_t dma_desc; if (!is_sec1) { dma_sync_single_for_cpu(dev, request->dma_desc, @@ -334,19 +359,17 @@ static __be32 get_request_hdr(struct device *dev, return request->desc->hdr; } - if (!request->desc->next_desc) { - dma_sync_single_for_cpu(dev, request->dma_desc, - TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); - return request->desc->hdr1; - } else { - dma_sync_single_for_cpu(dev, - be32_to_cpu(request->desc->next_desc), - TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); - edesc = container_of(request->desc, struct talitos_edesc, desc); - - return ((struct talitos_desc *)(edesc->buf + edesc->dma_len)) - ->hdr1; + edesc = container_of(request->desc, struct talitos_edesc, desc); + dma_desc = request->dma_desc; + while (edesc->next_desc) { + dma_desc = be32_to_cpu(edesc->desc.next_desc); + edesc = edesc->next_desc; } + + dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + return edesc->desc.hdr1; } /* @@ -356,6 +379,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request, saved_req; + struct talitos_edesc *edesc; unsigned long flags; int tail, status; bool is_sec1 = has_ftr_sec1(priv); @@ -380,9 +404,22 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) else status = error; - dma_unmap_single(dev, request->dma_desc, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); + if (is_sec1) { + dma_unmap_single(dev, request->dma_desc, + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + edesc = container_of(request->desc, + struct talitos_edesc, desc); + while (edesc->next_desc) { + dma_unmap_single( + dev, be32_to_cpu(edesc->desc.next_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + edesc = edesc->next_desc; + } + } else { + dma_unmap_single(dev, request->dma_desc, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } /* copy entries so we can call callback outside lock */ saved_req.desc = request->desc; @@ -477,8 +514,12 @@ DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) static __be32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + struct talitos_request *request; + struct talitos_edesc *edesc; int tail, iter; dma_addr_t cur_desc; + __be32 hdr = 0; cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); @@ -489,27 +530,35 @@ static __be32 current_desc_hdr(struct device *dev, int ch) } tail = priv->chan[ch].tail; - iter = tail; - while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && - priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { - iter = (iter + 1) & (priv->fifo_len - 1); - if (iter == tail) { - dev_err(dev, "couldn't locate current descriptor\n"); - return 0; + do { + request = &priv->chan[ch].fifo[iter]; + + if (request->dma_desc == cur_desc) { + hdr = request->desc->hdr; + } else if (is_sec1) { + edesc = container_of(request->desc, + struct talitos_edesc, desc); + while (edesc->next_desc) { + if (edesc->desc.next_desc == + cpu_to_be32(cur_desc)) { + hdr = edesc->next_desc->desc.hdr1; + break; + } + edesc = edesc->next_desc; + } } - } - if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { - struct talitos_edesc *edesc; + if (hdr) + break; - edesc = container_of(priv->chan[ch].fifo[iter].desc, - struct talitos_edesc, desc); - return ((struct talitos_desc *) - (edesc->buf + edesc->dma_len))->hdr; - } + iter = (iter + 1) & (priv->fifo_len - 1); + } while (iter != tail); + + if (!hdr) + dev_err(dev, "couldn't locate current descriptor\n"); - return priv->chan[ch].fifo[iter].desc->hdr; + return hdr; } /* @@ -1408,10 +1457,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, dma_len = 0; } alloc_len += icv_stashing ? authsize : 0; - - /* if its a ahash, add space for a second desc next to the first one */ - if (is_sec1 && !dst) - alloc_len += sizeof(struct talitos_desc); alloc_len += ivsize; edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); @@ -1427,6 +1472,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; + edesc->next_desc = NULL; if (dma_len) edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], edesc->dma_len, @@ -1727,8 +1773,10 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); struct talitos_desc *desc = &edesc->desc; - struct talitos_desc *desc2 = (struct talitos_desc *) - (edesc->buf + edesc->dma_len); + struct talitos_desc *desc2; + + if (desc->next_desc) + desc2 = &edesc->next_desc->desc; unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); if (desc->next_desc && @@ -1756,10 +1804,17 @@ static void common_nonsnoop_hash_unmap(struct device *dev, if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); +} - if (desc->next_desc) - dma_unmap_single(dev, be32_to_cpu(desc->next_desc), - TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); +static void free_edesc_list_from(struct talitos_edesc *edesc) +{ + struct talitos_edesc *next; + + while (edesc) { + next = edesc->next_desc; + kfree(edesc); + edesc = next; + } } static void ahash_done(struct device *dev, @@ -1778,7 +1833,7 @@ static void ahash_done(struct device *dev, } common_nonsnoop_hash_unmap(dev, edesc, areq); - kfree(edesc); + free_edesc_list_from(edesc); if (err) { ahash_request_complete(areq, err); @@ -1894,14 +1949,23 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); if (is_sec1 && req_ctx->nbuf && length) { - struct talitos_desc *desc2 = (struct talitos_desc *) - (edesc->buf + edesc->dma_len); - dma_addr_t next_desc; + struct talitos_edesc *edesc2; + struct talitos_desc *desc2; + + edesc2 = kzalloc(sizeof(*edesc2), + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : + GFP_ATOMIC); + if (!edesc2) { + ret = -ENOMEM; + goto err; + } + edesc->next_desc = edesc2; + + desc2 = &edesc2->desc; - memset(desc2, 0, sizeof(*desc2)); desc2->hdr = desc->hdr; desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; - desc2->hdr1 = desc2->hdr; desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; desc->hdr &= ~DESC_HDR_DONE_NOTIFY; @@ -1925,21 +1989,21 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, req_ctx->hw_context_size, req_ctx->hw_context, DMA_FROM_DEVICE); - - next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - desc->next_desc = cpu_to_be32(next_desc); } if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - ret = talitos_submit(dev, ctx->ch, desc, callback, areq); - if (ret != -EINPROGRESS) { - common_nonsnoop_hash_unmap(dev, edesc, areq); - kfree(edesc); - } + ret = talitos_submit(dev, ctx->ch, desc, callback, + areq); + if (ret != -EINPROGRESS) + goto err; + + return -EINPROGRESS; +err: + common_nonsnoop_hash_unmap(dev, edesc, areq); + kfree(edesc); return ret; } diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 34b0b5fab7e76..1553afa57b0be 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -49,6 +49,7 @@ struct talitos_desc { * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl/buf + * @next_desc: next descriptor * @desc: h/w descriptor * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) @@ -63,6 +64,7 @@ struct talitos_edesc { dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; + struct talitos_edesc *next_desc; struct talitos_desc desc; union { DECLARE_FLEX_ARRAY(struct talitos_ptr, link_tbl);