crypto: scatterwalk - add new functions for copying data

Add memcpy_from_sglist() and memcpy_to_sglist() which are more readable
versions of scatterwalk_map_and_copy() with the 'out' argument 0 and 1
respectively.  They follow the same argument order as memcpy_from_page()
and memcpy_to_page() from <linux/highmem.h>.  Note that in the case of
memcpy_from_sglist(), this also happens to be the same argument order
that scatterwalk_map_and_copy() uses.

The new code is also faster, mainly because it builds the scatter_walk
directly without creating a temporary scatterlist.  E.g., a 20%
performance improvement is seen for copying the AES-GCM auth tag.

Make scatterwalk_map_and_copy() be a wrapper around memcpy_from_sglist()
and memcpy_to_sglist().  Callers of scatterwalk_map_and_copy() should be
updated to call memcpy_from_sglist() or memcpy_to_sglist() directly, but
there are a lot of them so they aren't all being updated right away.

Also add functions memcpy_from_scatterwalk() and memcpy_to_scatterwalk()
which are similar but operate on a scatter_walk instead of a
scatterlist.  These will replace scatterwalk_copychunks() with the 'out'
argument 0 and 1 respectively.  Their behavior differs slightly from
scatterwalk_copychunks() in that they automatically take care of
flushing the dcache when needed, making them easier to use.

scatterwalk_copychunks() itself is left unchanged for now.  It will be
removed after its callers are updated to use other functions instead.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c
index af436ad02e3fff46e704192db5771e9aa29525e4..2e7a532152d61f204fd4baaeefaac81d2b991cd0 100644 (file)
--- a/crypto/scatterwalk.c
+++ b/crypto/scatterwalk.c
@@ -67,22 +67,63 @@ void scatterwalk_copychunks(void *buf, struct scatter_walk *walk,
 }
 EXPORT_SYMBOL_GPL(scatterwalk_copychunks);
 
-void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
-                             unsigned int start, unsigned int nbytes, int out)
+inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk,
+                                   unsigned int nbytes)
+{
+       do {
+               const void *src_addr;
+               unsigned int to_copy;
+
+               src_addr = scatterwalk_next(walk, nbytes, &to_copy);
+               memcpy(buf, src_addr, to_copy);
+               scatterwalk_done_src(walk, src_addr, to_copy);
+               buf += to_copy;
+               nbytes -= to_copy;
+       } while (nbytes);
+}
+EXPORT_SYMBOL_GPL(memcpy_from_scatterwalk);
+
+inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf,
+                                 unsigned int nbytes)
+{
+       do {
+               void *dst_addr;
+               unsigned int to_copy;
+
+               dst_addr = scatterwalk_next(walk, nbytes, &to_copy);
+               memcpy(dst_addr, buf, to_copy);
+               scatterwalk_done_dst(walk, dst_addr, to_copy);
+               buf += to_copy;
+               nbytes -= to_copy;
+       } while (nbytes);
+}
+EXPORT_SYMBOL_GPL(memcpy_to_scatterwalk);
+
+void memcpy_from_sglist(void *buf, struct scatterlist *sg,
+                       unsigned int start, unsigned int nbytes)
 {
        struct scatter_walk walk;
-       struct scatterlist tmp[2];
 
-       if (!nbytes)
+       if (unlikely(nbytes == 0)) /* in case sg == NULL */
                return;
 
-       sg = scatterwalk_ffwd(tmp, sg, start);
+       scatterwalk_start_at_pos(&walk, sg, start);
+       memcpy_from_scatterwalk(buf, &walk, nbytes);
+}
+EXPORT_SYMBOL_GPL(memcpy_from_sglist);
+
+void memcpy_to_sglist(struct scatterlist *sg, unsigned int start,
+                     const void *buf, unsigned int nbytes)
+{
+       struct scatter_walk walk;
+
+       if (unlikely(nbytes == 0)) /* in case sg == NULL */
+               return;
 
-       scatterwalk_start(&walk, sg);
-       scatterwalk_copychunks(buf, &walk, nbytes, out);
-       scatterwalk_done(&walk, out, 0);
+       scatterwalk_start_at_pos(&walk, sg, start);
+       memcpy_to_scatterwalk(&walk, buf, nbytes);
 }
-EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy);
+EXPORT_SYMBOL_GPL(memcpy_to_sglist);
 
 struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],
                                     struct scatterlist *src,

diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h
index 8e83c43016c9d4399356d7ea46a1f4a66ffde252..1689ecd7ddafa484428123dd990d5ddb807634cb 100644 (file)
--- a/include/crypto/scatterwalk.h
+++ b/include/crypto/scatterwalk.h
@@ -170,8 +170,28 @@ void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes);
 void scatterwalk_copychunks(void *buf, struct scatter_walk *walk,
                            size_t nbytes, int out);
 
-void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
-                             unsigned int start, unsigned int nbytes, int out);
+void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk,
+                            unsigned int nbytes);
+
+void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf,
+                          unsigned int nbytes);
+
+void memcpy_from_sglist(void *buf, struct scatterlist *sg,
+                       unsigned int start, unsigned int nbytes);
+
+void memcpy_to_sglist(struct scatterlist *sg, unsigned int start,
+                     const void *buf, unsigned int nbytes);
+
+/* In new code, please use memcpy_{from,to}_sglist() directly instead. */
+static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
+                                           unsigned int start,
+                                           unsigned int nbytes, int out)
+{
+       if (out)
+               memcpy_to_sglist(sg, start, buf, nbytes);
+       else
+               memcpy_from_sglist(buf, sg, start, nbytes);
+}
 
 struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],
                                     struct scatterlist *src,