Add Adler32 ARM NEON DotProd variant

Introduce a dotprod-accelerated Adler-32 using vdotq_u32 for both the
byte sum (s1) and the position-weighted sum (s2). Four independent
accumulator sets break the dependency chains between successive dotprod
instructions, allowing the pipeline to fill without stalling on results
from the prior iteration.

On Apple M3, the dotprod path is roughly 49% faster than the standard
NEON implementation for inputs above 32 bytes.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 17ac696f06c623a0bfe2840d3a6df94b45f47381..1b080ef1d1dac16b1ab1057bc69a72028b3b7fa3 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -732,6 +732,15 @@ if(WITH_OPTIM)
                 if(NEON_HAS_LD4)
                     add_definitions(-DARM_NEON_HASLD4)
                 endif()
+
+                check_neon_dotprod_compiler_flag()
+                if(NEON_HAS_DOTPROD AND ARCH_64BIT)
+                    add_definitions(-DARM_NEON_DOTPROD)
+                    set(NEON_DOTPROD_SRCS ${ARCHDIR}/adler32_neon_dotprod.c)
+                    list(APPEND ZLIB_ARCH_SRCS ${NEON_DOTPROD_SRCS})
+                    set_property(SOURCE ${NEON_DOTPROD_SRCS} PROPERTY COMPILE_FLAGS "${NEONDOTPRODFLAG} ${NOLTOFLAG}")
+                    add_feature_info(NEON_DOTPROD_ADLER32 1 "Support NEON DotProd instructions in adler32, using \"${NEONDOTPRODFLAG}\"")
+                endif()
             else()
                 set(WITH_NEON OFF)
             endif()

diff --git a/README.md b/README.md
index 0e99ed2da9bdfa82139b703db5ec5ee740082d6b..26866ea16c1001a2cec0338d1bb9a042e5bb8310 100644 (file)
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ Features
 * Modern C11 syntax and a clean code layout
 * Deflate medium and quick algorithms based on Intel’s zlib fork
 * Support for CPU intrinsics when available
-  * Adler32 implementation using SSSE3, SSE4.2, AVX2, AVX512, AVX512-VNNI, Neon, VMX & VSX, LSX, LASX, RVV
+  * Adler32 implementation using SSSE3, SSE4.2, AVX2, AVX512, AVX512-VNNI, Neon, Neon(DotProd), VMX & VSX, LSX, LASX, RVV
   * CRC32-B implementation using SSE2, SSE4.1, (V)PCLMULQDQ, ARMv8, ARMv8.2 PMULL+EOR3, Power8, IBM Z, LoongArch, ZBC
   * Slide hash implementations using SSE2, AVX2, ARMv6, Neon, Power8, VMX & VSX, LSX, LASX
   * Compare256 implementations using SSE2, AVX2, AVX512, Neon, Power9, LSX, LASX, RVV

diff --git a/arch/arm/Makefile.in b/arch/arm/Makefile.in
index d0bfe0e172719649ccd636aba6d193f6f005e4b8..d29182d02aadfee9b8f4dd7a09b74df5c9631d27 100644 (file)
--- a/arch/arm/Makefile.in
+++ b/arch/arm/Makefile.in
@@ -11,6 +11,7 @@ SUFFIX=
 ARMV8FLAG=
 PMULLEOR3FLAG=
 NEONFLAG=
+NEONDOTPRODFLAG=
 ARMV6FLAG=
 NOLTOFLAG=
 
@@ -20,6 +21,7 @@ TOPDIR=$(SRCTOP)
 
 all: \
        adler32_neon.o adler32_neon.lo \
+       adler32_neon_dotprod.o adler32_neon_dotprod.lo \
        arm_features.o arm_features.lo \
        chunkset_neon.o chunkset_neon.lo \
        compare256_neon.o compare256_neon.lo \
@@ -34,6 +36,12 @@ adler32_neon.o:
 adler32_neon.lo:
        $(CC) $(SFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon.c
 
+adler32_neon_dotprod.o:
+       $(CC) $(CFLAGS) $(NEONDOTPRODFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon_dotprod.c
+
+adler32_neon_dotprod.lo:
+       $(CC) $(SFLAGS) $(NEONDOTPRODFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon_dotprod.c
+
 arm_features.o:
        $(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/arm_features.c
 

diff --git a/arch/arm/adler32_neon_dotprod.c b/arch/arm/adler32_neon_dotprod.c
new file mode 100644 (file)
index 0000000..236fd5c
--- /dev/null
+++ b/arch/arm/adler32_neon_dotprod.c
@@ -0,0 +1,31 @@
+/* Copyright (C) 1995-2011, 2016 Mark Adler
+ * Copyright (C) 2017 ARM Holdings Inc.
+ * Copyright (C) 2025 Nathan Moinvaziri
+ * Authors:
+ *   Adenilson Cavalcanti <adenilson.cavalcanti@arm.com>
+ *   Adam Stylinski <kungfujesus06@gmail.com>
+ *   Nathan Moinvaziri <nathan@nathanm.com>
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+#if defined(ARM_NEON) && defined(ARM_NEON_DOTPROD)
+
+#define USE_DOTPROD
+#include "adler32_neon_tpl.h"
+
+Z_INTERNAL uint32_t adler32_neon_dotprod(uint32_t adler, const uint8_t *src, size_t len) {
+    return adler32_copy_impl(adler, NULL, src, len, 0);
+}
+
+Z_INTERNAL uint32_t adler32_copy_neon_dotprod(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
+#if OPTIMAL_CMP >= 32
+    return adler32_copy_impl(adler, dst, src, len, 1);
+#else
+    /* Without unaligned access, interleaved stores get decomposed into byte ops */
+    adler = adler32_neon_dotprod(adler, src, len);
+    memcpy(dst, src, len);
+    return adler;
+#endif
+}
+
+#endif

diff --git a/arch/arm/adler32_neon_tpl.h b/arch/arm/adler32_neon_tpl.h
index add12d4eca5f67ec85415f0c5a7a7992f3ae1780..57f52a72b6498701b57d1302b65999da0c651439 100644 (file)
--- a/arch/arm/adler32_neon_tpl.h
+++ b/arch/arm/adler32_neon_tpl.h
@@ -1,8 +1,10 @@
 /* Copyright (C) 1995-2011, 2016 Mark Adler
  * Copyright (C) 2017 ARM Holdings Inc.
+ * Copyright (C) 2025 Nathan Moinvaziri
  * Authors:
  *   Adenilson Cavalcanti <adenilson.cavalcanti@arm.com>
  *   Adam Stylinski <kungfujesus06@gmail.com>
+ *   Nathan Moinvaziri <nathan@nathanm.com>
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -10,7 +12,12 @@
 #include "neon_intrins.h"
 #include "adler32_p.h"
 
+#ifdef USE_DOTPROD
+/* Multiplication table for dotprod - must be uint8_t for vdotq_u32 */
+static const uint8_t ALIGNED_(64) taps[64] = {
+#else
 static const uint16_t ALIGNED_(64) taps[64] = {
+#endif
     64, 63, 62, 61, 60, 59, 58, 57,
     56, 55, 54, 53, 52, 51, 50, 49,
     48, 47, 46, 45, 44, 43, 42, 41,
@@ -69,11 +76,36 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
     while (len >= 16) {
         n = MIN(len, n);
 
+#ifdef USE_DOTPROD
+        /* Use 4 independent accumulator sets to break dependency chains
+         * and allow better instruction-level parallelism */
+        uint32x4_t adacc_a = vdupq_n_u32(0);
+        uint32x4_t adacc_b = vdupq_n_u32(0);
+        uint32x4_t adacc_c = vdupq_n_u32(0);
+        uint32x4_t adacc_d = vdupq_n_u32(0);
+        uint32x4_t s2acc_a = vdupq_n_u32(0);
+        uint32x4_t s2acc_b = vdupq_n_u32(0);
+        uint32x4_t s2acc_c = vdupq_n_u32(0);
+        uint32x4_t s2acc_d = vdupq_n_u32(0);
+        uint32x4_t s1sums_a = vdupq_n_u32(0);
+        uint32x4_t s1sums_b = vdupq_n_u32(0);
+        uint32x4_t s1sums_c = vdupq_n_u32(0);
+        uint32x4_t s1sums_d = vdupq_n_u32(0);
+
+        adacc_a = vsetq_lane_u32(pair[0], adacc_a, 0);
+        s2acc_a = vsetq_lane_u32(pair[1], s2acc_a, 0);
+
+        /* Load multiplication tables as uint8x16_t for dotprod */
+        uint8x16_t t0 = vld1q_u8(taps);
+        uint8x16_t t1 = vld1q_u8(taps + 16);
+        uint8x16_t t2 = vld1q_u8(taps + 32);
+        uint8x16_t t3 = vld1q_u8(taps + 48);
+
+        /* Vector of ones for s1 accumulation */
+        uint8x16_t ones = vdupq_n_u8(1);
+#else
         uint32x4_t adacc = vdupq_n_u32(0);
         uint32x4_t s2acc = vdupq_n_u32(0);
-        uint32x4_t s2acc_0 = vdupq_n_u32(0);
-        uint32x4_t s2acc_1 = vdupq_n_u32(0);
-        uint32x4_t s2acc_2 = vdupq_n_u32(0);
 
         adacc = vsetq_lane_u32(pair[0], adacc, 0);
         s2acc = vsetq_lane_u32(pair[1], s2acc, 0);
@@ -81,11 +113,16 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
         uint32x4_t s3acc = vdupq_n_u32(0);
         uint32x4_t adacc_prev = adacc;
 
+        uint32x4_t s2acc_0 = vdupq_n_u32(0);
+        uint32x4_t s2acc_1 = vdupq_n_u32(0);
+        uint32x4_t s2acc_2 = vdupq_n_u32(0);
+
         uint16x8_t s2_0, s2_1, s2_2, s2_3;
         s2_0 = s2_1 = s2_2 = s2_3 = vdupq_n_u16(0);
 
         uint16x8_t s2_4, s2_5, s2_6, s2_7;
         s2_4 = s2_5 = s2_6 = s2_7 = vdupq_n_u16(0);
+#endif
 
         size_t num_iter = (n >> 4) >> 2;
         int rem = (n >> 4) & 3;
@@ -114,6 +151,26 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
                 d3 = d0_d3.val[3];
             }
 
+#ifdef USE_DOTPROD
+            /* Each 16-byte chunk uses its own accumulator set so that
+             * successive dotprod instructions are independent and can
+             * be pipelined without stalling on the previous result */
+            s1sums_a = vaddq_u32(s1sums_a, adacc_a);
+            adacc_a = vdotq_u32(adacc_a, d0, ones);
+            s2acc_a = vdotq_u32(s2acc_a, d0, t0);
+
+            s1sums_b = vaddq_u32(s1sums_b, adacc_b);
+            adacc_b = vdotq_u32(adacc_b, d1, ones);
+            s2acc_b = vdotq_u32(s2acc_b, d1, t1);
+
+            s1sums_c = vaddq_u32(s1sums_c, adacc_c);
+            adacc_c = vdotq_u32(adacc_c, d2, ones);
+            s2acc_c = vdotq_u32(s2acc_c, d2, t2);
+
+            s1sums_d = vaddq_u32(s1sums_d, adacc_d);
+            adacc_d = vdotq_u32(adacc_d, d3, ones);
+            s2acc_d = vdotq_u32(s2acc_d, d3, t3);
+#else
             /* Unfortunately it doesn't look like there's a direct sum 8 bit to 32
              * bit instruction, we'll have to make due summing to 16 bits first */
             uint16x8x2_t hsum, hsum_fold;
@@ -141,12 +198,25 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
             s2_5 = vaddw_high_u8(s2_5, d2);
             s2_6 = vaddw_u8(s2_6, vget_low_u8(d3));
             s2_7 = vaddw_high_u8(s2_7, d3);
+#endif
 
+#ifndef USE_DOTPROD
             adacc_prev = adacc;
+#endif
             src += 64;
         }
 
+#ifdef USE_DOTPROD
+        /* Combine 4 independent accumulator sets into single vectors
+         * for the remainder loop and final reduction */
+        uint32x4_t adacc = vaddq_u32(vaddq_u32(adacc_a, adacc_b), vaddq_u32(adacc_c, adacc_d));
+        uint32x4_t s2acc = vaddq_u32(vaddq_u32(s2acc_a, s2acc_b), vaddq_u32(s2acc_c, s2acc_d));
+        uint32x4_t s1sums = vaddq_u32(vaddq_u32(s1sums_a, s1sums_b), vaddq_u32(s1sums_c, s1sums_d));
+        uint32x4_t s3acc = vshlq_n_u32(s1sums, 6);
+        uint32x4_t adacc_prev = adacc;
+#else
         s3acc = vshlq_n_u32(s3acc, 6);
+#endif
 
         if (rem) {
             uint32x4_t s3acc_0 = vdupq_n_u32(0);
@@ -156,12 +226,20 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
                     vst1q_u8(dst, d0);
                     dst += 16;
                 }
+
+#ifdef USE_DOTPROD
+                s3acc_0 = vaddq_u32(s3acc_0, adacc_prev);
+                adacc = vdotq_u32(adacc, d0, ones);
+                s2acc = vdotq_u32(s2acc, d0, t3);
+#else
                 uint16x8_t hsum;
                 hsum = vpaddlq_u8(d0);
                 s2_6 = vaddw_u8(s2_6, vget_low_u8(d0));
                 s2_7 = vaddw_high_u8(s2_7, d0);
                 adacc = vpadalq_u16(adacc, hsum);
                 s3acc_0 = vaddq_u32(s3acc_0, adacc_prev);
+#endif
+
                 adacc_prev = adacc;
                 src += 16;
             }
@@ -170,6 +248,12 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
             s3acc = vaddq_u32(s3acc_0, s3acc);
         }
 
+#ifdef USE_DOTPROD
+        /* Dotprod computes weighted sums inline, so final reduction is simple */
+        s2acc = vaddq_u32(s2acc, s3acc);
+        pair[0] = vaddvq_u32(adacc);
+        pair[1] = vaddvq_u32(s2acc);
+#else
         uint16x8x4_t t0_t3 = vld1q_u16_x4_ex(taps, 256);
         uint16x8x4_t t4_t7 = vld1q_u16_x4_ex(taps + 32, 256);
 
@@ -200,6 +284,7 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co
         s2acc = vaddq_u32(s2acc, s3acc);
         pair[0] = vaddvq_u32(adacc);
         pair[1] = vaddvq_u32(s2acc);
+#endif
 
         pair[0] %= BASE;
         pair[1] %= BASE;

diff --git a/arch/arm/arm_features.c b/arch/arm/arm_features.c
index 8f179526efd5144b26215f0720ac9c3e28ce9986..60ff0220b75b16556f935905b6976ed0445dd209 100644 (file)
--- a/arch/arm/arm_features.c
+++ b/arch/arm/arm_features.c
@@ -174,6 +174,48 @@ static int arm_has_eor3(void) {
     return has_eor3;
 }
 
+static int arm_has_dotprod(void) {
+    int has_dotprod = 0;
+#if defined(__ARM_FEATURE_DOTPROD)
+    /* Compile-time check */
+    has_dotprod = 1;
+#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
+#  ifdef HWCAP_ASIMDDP
+    has_dotprod = (getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0;
+#  endif
+#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
+#  ifdef HWCAP_ASIMDDP
+    unsigned long hwcap = 0;
+    elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
+    has_dotprod = (hwcap & HWCAP_ASIMDDP) != 0;
+#  endif
+#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
+#  ifdef ID_AA64ISAR0_DP_VAL
+    has_dotprod = getenv("QEMU_EMULATING") == NULL
+      && ID_AA64ISAR0_DP_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_DP_IMPL;
+#  endif
+#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
+#  ifdef ID_AA64ISAR0_DP
+    int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
+    uint64_t isar0 = 0;
+    size_t len = sizeof(isar0);
+    if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
+      has_dotprod = ID_AA64ISAR0_DP(isar0) >= ID_AA64ISAR0_DP_IMPL;
+    }
+#  endif
+#elif defined(__APPLE__)
+    int has_feat = 0;
+    size_t size = sizeof(has_feat);
+    has_dotprod = sysctlbyname("hw.optional.arm.FEAT_DotProd", &has_feat, &size, NULL, 0) == 0
+        && has_feat == 1;
+#elif defined(_WIN32)
+#  ifdef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
+    has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE);
+#  endif
+#endif
+    return has_dotprod;
+}
+
 /* AArch64 has neon. */
 #ifdef ARCH_32BIT
 static inline int arm_has_neon(void) {
@@ -329,6 +371,7 @@ void Z_INTERNAL arm_check_features(struct arm_cpu_features *features) {
     features->has_pmull = arm_has_pmull();
     features->has_eor3 = arm_has_eor3();
     features->has_fast_pmull = features->has_pmull && arm_cpu_has_fast_pmull();
+    features->has_dotprod = arm_has_dotprod();
 }
 
 #endif

diff --git a/arch/arm/arm_features.h b/arch/arm/arm_features.h
index 2f17a9ddf0515c00dc09a9a6fcef5a4cee542f3f..28617130a7d91f56a6a02ca6e23b08bd6382dd11 100644 (file)
--- a/arch/arm/arm_features.h
+++ b/arch/arm/arm_features.h
@@ -9,6 +9,7 @@ struct arm_cpu_features {
     int has_simd;
     int has_neon;
     int has_crc32;
+    int has_dotprod;
     int has_pmull;
     int has_eor3;
     int has_fast_pmull;

diff --git a/arch/arm/arm_functions.h b/arch/arm/arm_functions.h
index f2975fadfaeb1276d3777f7a13bcf521970ee43c..6eb72a3dac5441524a348367505550928a146622 100644 (file)
--- a/arch/arm/arm_functions.h
+++ b/arch/arm/arm_functions.h
@@ -18,6 +18,11 @@ uint32_t longest_match_roll_neon(deflate_state *const s, uint32_t cur_match);
 void slide_hash_neon(deflate_state *s);
 #endif
 
+#ifdef ARM_NEON_DOTPROD
+uint32_t adler32_neon_dotprod(uint32_t adler, const uint8_t *buf, size_t len);
+uint32_t adler32_copy_neon_dotprod(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
+#endif
+
 #ifndef ARM_NEON_NATIVE
 #  define ADLER32_FALLBACK
 #  define CHUNKSET_FALLBACK
@@ -70,6 +75,13 @@ void slide_hash_armv6(deflate_state *s);
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_neon
 #  endif
+// ARM - NEON DotProd
+#  ifdef ARM_NEON_DOTPROD_NATIVE
+#    undef native_adler32
+#    define native_adler32 adler32_neon_dotprod
+#    undef native_adler32_copy
+#    define native_adler32_copy adler32_copy_neon_dotprod
+#  endif
 // ARM - CRC32
 #  ifdef ARM_CRC32_NATIVE
 #    undef native_crc32

diff --git a/arch/arm/arm_natives.h b/arch/arm/arm_natives.h
index 311e33e958ad8cf459909799139603153eb9b1ec..7556577d474d89d61c51aa4bca0a4859369fa1fa 100644 (file)
--- a/arch/arm/arm_natives.h
+++ b/arch/arm/arm_natives.h
@@ -16,6 +16,12 @@
 #    define ARM_NEON_NATIVE
 #  endif
 #endif
+/* DotProd is optional in ARMv8.2+, mandatory in ARMv8.4+ */
+#if defined(__ARM_FEATURE_DOTPROD)
+#  ifdef ARM_NEON_DOTPROD
+#    define ARM_NEON_DOTPROD_NATIVE
+#  endif
+#endif
 /* CRC32 is optional in ARMv8.0, mandatory in ARMv8.1+ */
 #if defined(__ARM_FEATURE_CRC32) || (defined(__ARM_ARCH) && __ARM_ARCH >= 801)
 #  ifdef ARM_CRC32

diff --git a/cmake/detect-intrinsics.cmake b/cmake/detect-intrinsics.cmake
index 3b9acfa13a3631f24ad51d60eec09aca55943952..9d15ad23cd460a4f914bd47ee5bd3bf2a733ab11 100644 (file)
--- a/cmake/detect-intrinsics.cmake
+++ b/cmake/detect-intrinsics.cmake
@@ -334,6 +334,30 @@ macro(check_neon_ld4_intrinsics)
     set(CMAKE_REQUIRED_FLAGS)
 endmacro()
 
+macro(check_neon_dotprod_compiler_flag)
+    if(NOT NATIVEFLAG)
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang")
+            set(NEONDOTPRODFLAG "-march=armv8.2-a+dotprod")
+        endif()
+    endif()
+    set(CMAKE_REQUIRED_FLAGS "${NEONDOTPRODFLAG} ${NATIVEFLAG} ${ZNOLTOFLAG}")
+    check_c_source_compiles(
+        "#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_ARM64) || defined(_M_ARM64EC))
+        #  include <arm64_neon.h>
+        #else
+        #  include <arm_neon.h>
+        #endif
+        int main(void) {
+            uint8x16_t a = vdupq_n_u8(1);
+            uint8x16_t b = vdupq_n_u8(2);
+            uint32x4_t c = vdupq_n_u32(0);
+            c = vdotq_u32(c, a, b);
+            return vgetq_lane_u32(c, 0);
+        }"
+        NEON_HAS_DOTPROD)
+    set(CMAKE_REQUIRED_FLAGS)
+endmacro()
+
 macro(check_pclmulqdq_intrinsics)
     if(NOT NATIVEFLAG)
         if(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang" OR CMAKE_C_COMPILER_ID MATCHES "IntelLLVM" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC")

diff --git a/configure b/configure
index 9c9cff56cabd8e2175ddcb1191a80f543da32b5f..14974bf8b058ba639fdb178d1e2d5377752a1a21 100755 (executable)
--- a/configure
+++ b/configure
@@ -129,6 +129,7 @@ lasxflag="-mlasx"
 armv8flag=
 pmulleor3flag=
 neonflag=
+neondotprodflag=
 rvvflag=
 rvvzbcflag=
 zbcflag=
@@ -1334,6 +1335,31 @@ EOF
     fi
 }
 
+check_neon_dotprod_compiler_flag() {
+    neondotprodflag="-march=armv8.2-a+dotprod"
+    cat > $test.c << EOF
+#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC))
+#  include <arm64_neon.h>
+#else
+#  include <arm_neon.h>
+#endif
+int main(void) {
+    uint8x16_t a = vdupq_n_u8(1);
+    uint8x16_t b = vdupq_n_u8(2);
+    uint32x4_t c = vdupq_n_u32(0);
+    c = vdotq_u32(c, a, b);
+    return vgetq_lane_u32(c, 0);
+}
+EOF
+    if try $CC -c $CFLAGS $neondotprodflag $test.c; then
+        NEON_HAS_DOTPROD=1
+        echo "Check whether compiler supports NEON dotprod intrinsics ... Yes." | tee -a configure.log
+    else
+        NEON_HAS_DOTPROD=0
+        echo "Check whether compiler supports NEON dotprod intrinsics ... No." | tee -a configure.log
+    fi
+}
+
 check_neon_ld4_intrinsics() {
     cat > $test.c << EOF
 #if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC))
@@ -2011,6 +2037,15 @@ EOF
 
                     ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_neon.o chunkset_neon.o compare256_neon.o slide_hash_neon.o"
                     ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_neon.lo chunkset_neon.lo compare256_neon.lo slide_hash_neon.lo"
+
+                    check_neon_dotprod_compiler_flag
+
+                    if test $NEON_HAS_DOTPROD -eq 1 && test $ARCH_64BIT -eq 1; then
+                        CFLAGS="${CFLAGS} -DARM_NEON_DOTPROD"
+                        SFLAGS="${SFLAGS} -DARM_NEON_DOTPROD"
+                        ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_neon_dotprod.o"
+                        ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_neon_dotprod.lo"
+                    fi
                 fi
             fi
 
@@ -2318,6 +2353,7 @@ echo xsaveflag = $xsaveflag >> configure.log
 echo armv8flag = $armv8flag >> configure.log
 echo pmulleor3flag = $pmulleor3flag >> configure.log
 echo neonflag = $neonflag >> configure.log
+echo neondotprodflag = $neondotprodflag >> configure.log
 echo armv6flag = $armv6flag >> configure.log
 echo lsxflag = $lsxflag >> configure.log
 echo lasxflag = $lasxflag >> configure.log
@@ -2464,6 +2500,7 @@ sed < $SRCDIR/$ARCHDIR/Makefile.in "
 /^ARMV8FLAG *=/s#=.*#=$armv8flag#
 /^PMULLEOR3FLAG *=/s#=.*#=$pmulleor3flag#
 /^NEONFLAG *=/s#=.*#=$neonflag#
+/^NEONDOTPRODFLAG *=/s#=.*#=$neondotprodflag#
 /^ARMV6FLAG *=/s#=.*#=$armv6flag#
 /^NOLTOFLAG *=/s#=.*#=$noltoflag#
 /^S390VXFLAG *=/s#=.*#=$s390vxflag#

diff --git a/functable.c b/functable.c
index 6ad05f32e11e8b71131759662077a2797e94d8fb..cf453e3867cdc924ef342205e202ab04e5c6f556 100644 (file)
--- a/functable.c
+++ b/functable.c
@@ -267,6 +267,16 @@ static int init_functable(void) {
         ft.longest_match_roll = &longest_match_roll_neon;
         ft.slide_hash = &slide_hash_neon;
     }
+#endif
+    // ARM - NEON DotProd
+#ifdef ARM_NEON_DOTPROD
+#  ifndef ARM_NEON_DOTPROD_NATIVE
+    if (cf.arm.has_neon && cf.arm.has_dotprod)
+#  endif
+    {
+        ft.adler32 = &adler32_neon_dotprod;
+        ft.adler32_copy = &adler32_copy_neon_dotprod;
+    }
 #endif
     // ARM - CRC32
 #if defined(ARM_CRC32) && !defined(ARM_PMULL_EOR3_NATIVE)

diff --git a/test/benchmarks/benchmark_adler32.cc b/test/benchmarks/benchmark_adler32.cc
index 8b0a6beef1e8d640a6c143625a1eff10acc481ad..ce28f41c32edf3345402346e257189f6409bb239 100644 (file)
--- a/test/benchmarks/benchmark_adler32.cc
+++ b/test/benchmarks/benchmark_adler32.cc
@@ -90,6 +90,9 @@ BENCHMARK_ADLER32(native, native_adler32, 1);
 #ifdef ARM_NEON
 BENCHMARK_ADLER32(neon, adler32_neon, test_cpu_features.arm.has_neon);
 #endif
+#ifdef ARM_NEON_DOTPROD
+BENCHMARK_ADLER32(neon_dotprod, adler32_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod);
+#endif
 
 #ifdef PPC_VMX
 BENCHMARK_ADLER32(vmx, adler32_vmx, test_cpu_features.power.has_altivec);

diff --git a/test/benchmarks/benchmark_adler32_copy.cc b/test/benchmarks/benchmark_adler32_copy.cc
index 16526cd3fefd3c76657909e463fcbc93445791a2..610911f57e3512bbc2d4070170530f1c2b6e98f9 100644 (file)
--- a/test/benchmarks/benchmark_adler32_copy.cc
+++ b/test/benchmarks/benchmark_adler32_copy.cc
@@ -143,6 +143,9 @@ BENCHMARK_ADLER32_COPY(native, native_adler32, native_adler32_copy, 1);
 #ifdef ARM_NEON
 BENCHMARK_ADLER32_COPY(neon, adler32_neon, adler32_copy_neon, test_cpu_features.arm.has_neon);
 #endif
+#ifdef ARM_NEON_DOTPROD
+BENCHMARK_ADLER32_COPY(neon_dotprod, adler32_neon_dotprod, adler32_copy_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod);
+#endif
 
 #ifdef PPC_VMX
 BENCHMARK_ADLER32_COPY(vmx, adler32_vmx, adler32_copy_vmx, test_cpu_features.power.has_altivec);

diff --git a/test/test_adler32.cc b/test/test_adler32.cc
index 91189a4ae26d1333a1a1a42c088f3c0824ec4585..5924801756df43c79019f1127186142f91b3d2c4 100644 (file)
--- a/test/test_adler32.cc
+++ b/test/test_adler32.cc
@@ -47,7 +47,11 @@ TEST_ADLER32(native, native_adler32, 1)
 
 #ifdef ARM_NEON
 TEST_ADLER32(neon, adler32_neon, test_cpu_features.arm.has_neon)
-#elif defined(POWER8_VSX)
+#endif
+#ifdef ARM_NEON_DOTPROD
+TEST_ADLER32(neon_dotprod, adler32_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod)
+#endif
+#if defined(POWER8_VSX)
 TEST_ADLER32(power8, adler32_power8, test_cpu_features.power.has_arch_2_07)
 #elif defined(PPC_VMX)
 TEST_ADLER32(vmx, adler32_vmx, test_cpu_features.power.has_altivec)

diff --git a/test/test_adler32_copy.cc b/test/test_adler32_copy.cc
index b7ca376b7033f9c4f8dd2016f16bbe2223c9207d..b5cab369f30613abe6b99b28c777d7cb850fe2f8 100644 (file)
--- a/test/test_adler32_copy.cc
+++ b/test/test_adler32_copy.cc
@@ -52,7 +52,11 @@ TEST_ADLER32_COPY(c, adler32_copy_c, 1)
 
 #ifdef ARM_NEON
 TEST_ADLER32_COPY(neon, adler32_copy_neon, test_cpu_features.arm.has_neon)
-#elif defined(POWER8_VSX)
+#endif
+#ifdef ARM_NEON_DOTPROD
+TEST_ADLER32_COPY(neon_dotprod, adler32_copy_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod)
+#endif
+#if defined(POWER8_VSX)
 TEST_ADLER32_COPY(power8, adler32_copy_power8, test_cpu_features.power.has_arch_2_07)
 #elif defined(PPC_VMX)
 TEST_ADLER32_COPY(vmx, adler32_copy_vmx, test_cpu_features.power.has_altivec)