[arm] Add optimised string functions for 64-bit ARM

Signed-off-by: Michael Brown <mcb30@ipxe.org>

diff --git a/src/arch/arm/include/bits/string.h b/src/arch/arm32/include/bits/string.h
similarity index 100%
rename from src/arch/arm/include/bits/string.h
rename to src/arch/arm32/include/bits/string.h

diff --git a/src/arch/arm64/core/arm64_string.c b/src/arch/arm64/core/arm64_string.c
new file mode 100644 (file)
index 0000000..28a2b73
--- /dev/null
+++ b/src/arch/arm64/core/arm64_string.c
@@ -0,0 +1,249 @@
+/*
+ * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+/** @file
+ *
+ * Optimised string operations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+
+/**
+ * Copy memory area
+ *
+ * @v dest             Destination address
+ * @v src              Source address
+ * @v len              Length
+ * @ret dest           Destination address
+ */
+void arm64_memcpy ( void *dest, const void *src, size_t len ) {
+       void *discard_dest;
+       void *discard_end;
+       const void *discard_src;
+       size_t discard_offset;
+       unsigned long discard_data;
+       unsigned long discard_low;
+       unsigned long discard_high;
+
+       /* If length is too short for an "ldp"/"stp" instruction pair,
+        * then just copy individual bytes.
+        */
+       if ( len < 16 ) {
+               __asm__ __volatile__ ( "cbz %0, 2f\n\t"
+                                      "\n1:\n\t"
+                                      "sub %0, %0, #1\n\t"
+                                      "ldrb %w1, [%3, %0]\n\t"
+                                      "strb %w1, [%2, %0]\n\t"
+                                      "cbnz %0, 1b\n\t"
+                                      "\n2:\n\t"
+                                      : "=&r" ( discard_offset ),
+                                        "=&r" ( discard_data )
+                                      : "r" ( dest ), "r" ( src ), "0" ( len )
+                                      : "memory" );
+               return;
+       }
+
+       /* Use "ldp"/"stp" to copy 16 bytes at a time: one initial
+        * potentially unaligned access, multiple destination-aligned
+        * accesses, one final potentially unaligned access.
+        */
+       __asm__ __volatile__ ( "ldp %3, %4, [%1], #16\n\t"
+                              "stp %3, %4, [%0], #16\n\t"
+                              "and %3, %0, #15\n\t"
+                              "sub %0, %0, %3\n\t"
+                              "sub %1, %1, %3\n\t"
+                              "bic %2, %5, #15\n\t"
+                              "b 2f\n\t"
+                              "\n1:\n\t"
+                              "ldp %3, %4, [%1], #16\n\t"
+                              "stp %3, %4, [%0], #16\n\t"
+                              "\n2:\n\t"
+                              "cmp %0, %2\n\t"
+                              "bne 1b\n\t"
+                              "ldp %3, %4, [%6, #-16]\n\t"
+                              "stp %3, %4, [%5, #-16]\n\t"
+                              : "=&r" ( discard_dest ),
+                                "=&r" ( discard_src ),
+                                "=&r" ( discard_end ),
+                                "=&r" ( discard_low ),
+                                "=&r" ( discard_high )
+                              : "r" ( dest + len ), "r" ( src + len ),
+                                "0" ( dest ), "1" ( src )
+                              : "memory", "cc" );
+}
+
+/**
+ * Zero memory region
+ *
+ * @v dest             Destination region
+ * @v len              Length
+ */
+void arm64_bzero ( void *dest, size_t len ) {
+       size_t discard_offset;
+       void *discard_dest;
+       void *discard_end;
+
+       /* If length is too short for an "stp" instruction, then just
+        * zero individual bytes.
+        */
+       if ( len < 16 ) {
+               __asm__ __volatile__ ( "cbz %0, 2f\n\t"
+                                      "\n1:\n\t"
+                                      "sub %0, %0, #1\n\t"
+                                      "strb wzr, [%1, %0]\n\t"
+                                      "cbnz %0, 1b\n\t"
+                                      "\n2:\n\t"
+                                      : "=&r" ( discard_offset )
+                                      : "r" ( dest ), "0" ( len )
+                                      : "memory" );
+               return;
+       }
+
+       /* Use "stp" to zero 16 bytes at a time: one initial
+        * potentially unaligned access, multiple aligned accesses,
+        * one final potentially unaligned access.
+        */
+       __asm__ __volatile__ ( "stp xzr, xzr, [%0], #16\n\t"
+                              "bic %0, %0, #15\n\t"
+                              "bic %1, %2, #15\n\t"
+                              "b 2f\n\t"
+                              "\n1:\n\t"
+                              "stp xzr, xzr, [%0], #16\n\t"
+                              "\n2:\n\t"
+                              "cmp %0, %1\n\t"
+                              "bne 1b\n\t"
+                              "stp xzr, xzr, [%2, #-16]\n\t"
+                              : "=&r" ( discard_dest ),
+                                "=&r" ( discard_end )
+                              : "r" ( dest + len ), "0" ( dest )
+                              : "memory", "cc" );
+}
+
+/**
+ * Fill memory region
+ *
+ * @v dest             Destination region
+ * @v len              Length
+ * @v character                Fill character
+ *
+ * The unusual parameter order is to allow for more efficient
+ * tail-calling to arm64_memset() when zeroing a region.
+ */
+void arm64_memset ( void *dest, size_t len, int character ) {
+       size_t discard_offset;
+
+       /* Use optimised zeroing code if applicable */
+       if ( character == 0 ) {
+               arm64_bzero ( dest, len );
+               return;
+       }
+
+       /* Fill one byte at a time.  Calling memset() with a non-zero
+        * value is relatively rare and unlikely to be
+        * performance-critical.
+        */
+       __asm__ __volatile__ ( "cbz %0, 2f\n\t"
+                              "\n1:\n\t"
+                              "sub %0, %0, #1\n\t"
+                              "strb %w2, [%1, %0]\n\t"
+                              "cbnz %0, 1b\n\t"
+                              "\n2:\n\t"
+                              : "=&r" ( discard_offset )
+                              : "r" ( dest ), "r" ( character ), "0" ( len )
+                              : "memory" );
+}
+
+/**
+ * Copy (possibly overlapping) memory region forwards
+ *
+ * @v dest             Destination region
+ * @v src              Source region
+ * @v len              Length
+ */
+void arm64_memmove_forwards ( void *dest, const void *src, size_t len ) {
+       void *discard_dest;
+       const void *discard_src;
+       unsigned long discard_data;
+
+       /* Assume memmove() is not performance-critical, and perform a
+        * bytewise copy for simplicity.
+        */
+       __asm__ __volatile__ ( "b 2f\n\t"
+                              "\n1:\n\t"
+                              "ldrb %w2, [%1], #1\n\t"
+                              "strb %w2, [%0], #1\n\t"
+                              "\n2:\n\t"
+                              "cmp %0, %3\n\t"
+                              "bne 1b\n\t"
+                              : "=&r" ( discard_dest ),
+                                "=&r" ( discard_src ),
+                                "=&r" ( discard_data )
+                              : "r" ( dest + len ), "0" ( dest ), "1" ( src )
+                              : "memory" );
+}
+
+/**
+ * Copy (possibly overlapping) memory region backwards
+ *
+ * @v dest             Destination region
+ * @v src              Source region
+ * @v len              Length
+ */
+void arm64_memmove_backwards ( void *dest, const void *src, size_t len ) {
+       size_t discard_offset;
+       unsigned long discard_data;
+
+       /* Assume memmove() is not performance-critical, and perform a
+        * bytewise copy for simplicity.
+        */
+       __asm__ __volatile__ ( "cbz %0, 2f\n\t"
+                              "\n1:\n\t"
+                              "sub %0, %0, #1\n\t"
+                              "ldrb %w1, [%3, %0]\n\t"
+                              "strb %w1, [%2, %0]\n\t"
+                              "cbnz %0, 1b\n\t"
+                              "\n2:\n\t"
+                              : "=&r" ( discard_offset ),
+                                "=&r" ( discard_data )
+                              : "r" ( dest ), "r" ( src ), "0" ( len )
+                              : "memory" );
+}
+
+/**
+ * Copy (possibly overlapping) memory region
+ *
+ * @v dest             Destination region
+ * @v src              Source region
+ * @v len              Length
+ */
+void arm64_memmove ( void *dest, const void *src, size_t len ) {
+
+       if ( dest <= src ) {
+               arm64_memmove_forwards ( dest, src, len );
+       } else {
+               arm64_memmove_backwards ( dest, src, len );
+       }
+}

diff --git a/src/arch/arm64/include/bits/string.h b/src/arch/arm64/include/bits/string.h
new file mode 100644 (file)
index 0000000..c05fbe3
--- /dev/null
+++ b/src/arch/arm64/include/bits/string.h
@@ -0,0 +1,106 @@
+#ifndef BITS_STRING_H
+#define BITS_STRING_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * String functions
+ *
+ */
+
+extern void arm64_bzero ( void *dest, size_t len );
+extern void arm64_memset ( void *dest, size_t len, int character );
+extern void arm64_memcpy ( void *dest, const void *src, size_t len );
+extern void arm64_memmove_forwards ( void *dest, const void *src, size_t len );
+extern void arm64_memmove_backwards ( void *dest, const void *src, size_t len );
+extern void arm64_memmove ( void *dest, const void *src, size_t len );
+
+/**
+ * Fill memory region
+ *
+ * @v dest             Destination region
+ * @v character                Fill character
+ * @v len              Length
+ * @ret dest           Destination region
+ */
+static inline __attribute__ (( always_inline )) void *
+memset ( void *dest, int character, size_t len ) {
+
+       /* Allow gcc to generate inline "stX xzr" instructions for
+        * small, constant lengths.
+        */
+       if ( __builtin_constant_p ( character ) && ( character == 0 ) &&
+            __builtin_constant_p ( len ) && ( len <= 64 ) ) {
+               __builtin_memset ( dest, 0, len );
+               return dest;
+       }
+
+       /* For zeroing larger or non-constant lengths, use the
+        * optimised variable-length zeroing code.
+        */
+       if ( __builtin_constant_p ( character ) && ( character == 0 ) ) {
+               arm64_bzero ( dest, len );
+               return dest;
+       }
+
+       /* Not necessarily zeroing: use basic variable-length code */
+       arm64_memset ( dest, len, character );
+       return dest;
+}
+
+/**
+ * Copy memory region
+ *
+ * @v dest             Destination region
+ * @v src              Source region
+ * @v len              Length
+ * @ret dest           Destination region
+ */
+static inline __attribute__ (( always_inline )) void *
+memcpy ( void *dest, const void *src, size_t len ) {
+
+       /* Allow gcc to generate inline "ldX"/"stX" instructions for
+        * small, constant lengths.
+        */
+       if ( __builtin_constant_p ( len ) && ( len <= 64 ) ) {
+               __builtin_memcpy ( dest, src, len );
+               return dest;
+       }
+
+       /* Otherwise, use variable-length code */
+       arm64_memcpy ( dest, src, len );
+       return dest;
+}
+
+/**
+ * Copy (possibly overlapping) memory region
+ *
+ * @v dest             Destination region
+ * @v src              Source region
+ * @v len              Length
+ * @ret dest           Destination region
+ */
+static inline __attribute__ (( always_inline )) void *
+memmove ( void *dest, const void *src, size_t len ) {
+       ssize_t offset = ( dest - src );
+
+       /* If required direction of copy is known at build time, then
+        * use the appropriate forwards/backwards copy directly.
+        */
+       if ( __builtin_constant_p ( offset ) ) {
+               if ( offset <= 0 ) {
+                       arm64_memmove_forwards ( dest, src, len );
+                       return dest;
+               } else {
+                       arm64_memmove_backwards ( dest, src, len );
+                       return dest;
+               }
+       }
+
+       /* Otherwise, use ambidirectional copy */
+       arm64_memmove ( dest, src, len );
+       return dest;
+}
+
+#endif /* BITS_STRING_H */