From: Nathan Moinvaziri Date: Wed, 18 Mar 2026 16:41:49 +0000 (-0700) Subject: Add Adler32 ARM NEON DotProd variant X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d40f29fd42ed9158e3eb3e221dca50e4b627f7a8;p=thirdparty%2Fzlib-ng.git Add Adler32 ARM NEON DotProd variant Introduce a dotprod-accelerated Adler-32 using vdotq_u32 for both the byte sum (s1) and the position-weighted sum (s2). Four independent accumulator sets break the dependency chains between successive dotprod instructions, allowing the pipeline to fill without stalling on results from the prior iteration. On Apple M3, the dotprod path is roughly 49% faster than the standard NEON implementation for inputs above 32 bytes. Co-Authored-By: Claude Opus 4.6 (1M context) --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 17ac696f0..1b080ef1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -732,6 +732,15 @@ if(WITH_OPTIM) if(NEON_HAS_LD4) add_definitions(-DARM_NEON_HASLD4) endif() + + check_neon_dotprod_compiler_flag() + if(NEON_HAS_DOTPROD AND ARCH_64BIT) + add_definitions(-DARM_NEON_DOTPROD) + set(NEON_DOTPROD_SRCS ${ARCHDIR}/adler32_neon_dotprod.c) + list(APPEND ZLIB_ARCH_SRCS ${NEON_DOTPROD_SRCS}) + set_property(SOURCE ${NEON_DOTPROD_SRCS} PROPERTY COMPILE_FLAGS "${NEONDOTPRODFLAG} ${NOLTOFLAG}") + add_feature_info(NEON_DOTPROD_ADLER32 1 "Support NEON DotProd instructions in adler32, using \"${NEONDOTPRODFLAG}\"") + endif() else() set(WITH_NEON OFF) endif() diff --git a/README.md b/README.md index 0e99ed2da..26866ea16 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Features * Modern C11 syntax and a clean code layout * Deflate medium and quick algorithms based on Intel’s zlib fork * Support for CPU intrinsics when available - * Adler32 implementation using SSSE3, SSE4.2, AVX2, AVX512, AVX512-VNNI, Neon, VMX & VSX, LSX, LASX, RVV + * Adler32 implementation using SSSE3, SSE4.2, AVX2, AVX512, AVX512-VNNI, Neon, Neon(DotProd), VMX & VSX, LSX, LASX, RVV * CRC32-B implementation using SSE2, SSE4.1, (V)PCLMULQDQ, ARMv8, ARMv8.2 PMULL+EOR3, Power8, IBM Z, LoongArch, ZBC * Slide hash implementations using SSE2, AVX2, ARMv6, Neon, Power8, VMX & VSX, LSX, LASX * Compare256 implementations using SSE2, AVX2, AVX512, Neon, Power9, LSX, LASX, RVV diff --git a/arch/arm/Makefile.in b/arch/arm/Makefile.in index d0bfe0e17..d29182d02 100644 --- a/arch/arm/Makefile.in +++ b/arch/arm/Makefile.in @@ -11,6 +11,7 @@ SUFFIX= ARMV8FLAG= PMULLEOR3FLAG= NEONFLAG= +NEONDOTPRODFLAG= ARMV6FLAG= NOLTOFLAG= @@ -20,6 +21,7 @@ TOPDIR=$(SRCTOP) all: \ adler32_neon.o adler32_neon.lo \ + adler32_neon_dotprod.o adler32_neon_dotprod.lo \ arm_features.o arm_features.lo \ chunkset_neon.o chunkset_neon.lo \ compare256_neon.o compare256_neon.lo \ @@ -34,6 +36,12 @@ adler32_neon.o: adler32_neon.lo: $(CC) $(SFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon.c +adler32_neon_dotprod.o: + $(CC) $(CFLAGS) $(NEONDOTPRODFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon_dotprod.c + +adler32_neon_dotprod.lo: + $(CC) $(SFLAGS) $(NEONDOTPRODFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon_dotprod.c + arm_features.o: $(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/arm_features.c diff --git a/arch/arm/adler32_neon_dotprod.c b/arch/arm/adler32_neon_dotprod.c new file mode 100644 index 000000000..236fd5ca8 --- /dev/null +++ b/arch/arm/adler32_neon_dotprod.c @@ -0,0 +1,31 @@ +/* Copyright (C) 1995-2011, 2016 Mark Adler + * Copyright (C) 2017 ARM Holdings Inc. + * Copyright (C) 2025 Nathan Moinvaziri + * Authors: + * Adenilson Cavalcanti + * Adam Stylinski + * Nathan Moinvaziri + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#if defined(ARM_NEON) && defined(ARM_NEON_DOTPROD) + +#define USE_DOTPROD +#include "adler32_neon_tpl.h" + +Z_INTERNAL uint32_t adler32_neon_dotprod(uint32_t adler, const uint8_t *src, size_t len) { + return adler32_copy_impl(adler, NULL, src, len, 0); +} + +Z_INTERNAL uint32_t adler32_copy_neon_dotprod(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) { +#if OPTIMAL_CMP >= 32 + return adler32_copy_impl(adler, dst, src, len, 1); +#else + /* Without unaligned access, interleaved stores get decomposed into byte ops */ + adler = adler32_neon_dotprod(adler, src, len); + memcpy(dst, src, len); + return adler; +#endif +} + +#endif diff --git a/arch/arm/adler32_neon_tpl.h b/arch/arm/adler32_neon_tpl.h index add12d4ec..57f52a72b 100644 --- a/arch/arm/adler32_neon_tpl.h +++ b/arch/arm/adler32_neon_tpl.h @@ -1,8 +1,10 @@ /* Copyright (C) 1995-2011, 2016 Mark Adler * Copyright (C) 2017 ARM Holdings Inc. + * Copyright (C) 2025 Nathan Moinvaziri * Authors: * Adenilson Cavalcanti * Adam Stylinski + * Nathan Moinvaziri * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -10,7 +12,12 @@ #include "neon_intrins.h" #include "adler32_p.h" +#ifdef USE_DOTPROD +/* Multiplication table for dotprod - must be uint8_t for vdotq_u32 */ +static const uint8_t ALIGNED_(64) taps[64] = { +#else static const uint16_t ALIGNED_(64) taps[64] = { +#endif 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, @@ -69,11 +76,36 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co while (len >= 16) { n = MIN(len, n); +#ifdef USE_DOTPROD + /* Use 4 independent accumulator sets to break dependency chains + * and allow better instruction-level parallelism */ + uint32x4_t adacc_a = vdupq_n_u32(0); + uint32x4_t adacc_b = vdupq_n_u32(0); + uint32x4_t adacc_c = vdupq_n_u32(0); + uint32x4_t adacc_d = vdupq_n_u32(0); + uint32x4_t s2acc_a = vdupq_n_u32(0); + uint32x4_t s2acc_b = vdupq_n_u32(0); + uint32x4_t s2acc_c = vdupq_n_u32(0); + uint32x4_t s2acc_d = vdupq_n_u32(0); + uint32x4_t s1sums_a = vdupq_n_u32(0); + uint32x4_t s1sums_b = vdupq_n_u32(0); + uint32x4_t s1sums_c = vdupq_n_u32(0); + uint32x4_t s1sums_d = vdupq_n_u32(0); + + adacc_a = vsetq_lane_u32(pair[0], adacc_a, 0); + s2acc_a = vsetq_lane_u32(pair[1], s2acc_a, 0); + + /* Load multiplication tables as uint8x16_t for dotprod */ + uint8x16_t t0 = vld1q_u8(taps); + uint8x16_t t1 = vld1q_u8(taps + 16); + uint8x16_t t2 = vld1q_u8(taps + 32); + uint8x16_t t3 = vld1q_u8(taps + 48); + + /* Vector of ones for s1 accumulation */ + uint8x16_t ones = vdupq_n_u8(1); +#else uint32x4_t adacc = vdupq_n_u32(0); uint32x4_t s2acc = vdupq_n_u32(0); - uint32x4_t s2acc_0 = vdupq_n_u32(0); - uint32x4_t s2acc_1 = vdupq_n_u32(0); - uint32x4_t s2acc_2 = vdupq_n_u32(0); adacc = vsetq_lane_u32(pair[0], adacc, 0); s2acc = vsetq_lane_u32(pair[1], s2acc, 0); @@ -81,11 +113,16 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co uint32x4_t s3acc = vdupq_n_u32(0); uint32x4_t adacc_prev = adacc; + uint32x4_t s2acc_0 = vdupq_n_u32(0); + uint32x4_t s2acc_1 = vdupq_n_u32(0); + uint32x4_t s2acc_2 = vdupq_n_u32(0); + uint16x8_t s2_0, s2_1, s2_2, s2_3; s2_0 = s2_1 = s2_2 = s2_3 = vdupq_n_u16(0); uint16x8_t s2_4, s2_5, s2_6, s2_7; s2_4 = s2_5 = s2_6 = s2_7 = vdupq_n_u16(0); +#endif size_t num_iter = (n >> 4) >> 2; int rem = (n >> 4) & 3; @@ -114,6 +151,26 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co d3 = d0_d3.val[3]; } +#ifdef USE_DOTPROD + /* Each 16-byte chunk uses its own accumulator set so that + * successive dotprod instructions are independent and can + * be pipelined without stalling on the previous result */ + s1sums_a = vaddq_u32(s1sums_a, adacc_a); + adacc_a = vdotq_u32(adacc_a, d0, ones); + s2acc_a = vdotq_u32(s2acc_a, d0, t0); + + s1sums_b = vaddq_u32(s1sums_b, adacc_b); + adacc_b = vdotq_u32(adacc_b, d1, ones); + s2acc_b = vdotq_u32(s2acc_b, d1, t1); + + s1sums_c = vaddq_u32(s1sums_c, adacc_c); + adacc_c = vdotq_u32(adacc_c, d2, ones); + s2acc_c = vdotq_u32(s2acc_c, d2, t2); + + s1sums_d = vaddq_u32(s1sums_d, adacc_d); + adacc_d = vdotq_u32(adacc_d, d3, ones); + s2acc_d = vdotq_u32(s2acc_d, d3, t3); +#else /* Unfortunately it doesn't look like there's a direct sum 8 bit to 32 * bit instruction, we'll have to make due summing to 16 bits first */ uint16x8x2_t hsum, hsum_fold; @@ -141,12 +198,25 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co s2_5 = vaddw_high_u8(s2_5, d2); s2_6 = vaddw_u8(s2_6, vget_low_u8(d3)); s2_7 = vaddw_high_u8(s2_7, d3); +#endif +#ifndef USE_DOTPROD adacc_prev = adacc; +#endif src += 64; } +#ifdef USE_DOTPROD + /* Combine 4 independent accumulator sets into single vectors + * for the remainder loop and final reduction */ + uint32x4_t adacc = vaddq_u32(vaddq_u32(adacc_a, adacc_b), vaddq_u32(adacc_c, adacc_d)); + uint32x4_t s2acc = vaddq_u32(vaddq_u32(s2acc_a, s2acc_b), vaddq_u32(s2acc_c, s2acc_d)); + uint32x4_t s1sums = vaddq_u32(vaddq_u32(s1sums_a, s1sums_b), vaddq_u32(s1sums_c, s1sums_d)); + uint32x4_t s3acc = vshlq_n_u32(s1sums, 6); + uint32x4_t adacc_prev = adacc; +#else s3acc = vshlq_n_u32(s3acc, 6); +#endif if (rem) { uint32x4_t s3acc_0 = vdupq_n_u32(0); @@ -156,12 +226,20 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co vst1q_u8(dst, d0); dst += 16; } + +#ifdef USE_DOTPROD + s3acc_0 = vaddq_u32(s3acc_0, adacc_prev); + adacc = vdotq_u32(adacc, d0, ones); + s2acc = vdotq_u32(s2acc, d0, t3); +#else uint16x8_t hsum; hsum = vpaddlq_u8(d0); s2_6 = vaddw_u8(s2_6, vget_low_u8(d0)); s2_7 = vaddw_high_u8(s2_7, d0); adacc = vpadalq_u16(adacc, hsum); s3acc_0 = vaddq_u32(s3acc_0, adacc_prev); +#endif + adacc_prev = adacc; src += 16; } @@ -170,6 +248,12 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co s3acc = vaddq_u32(s3acc_0, s3acc); } +#ifdef USE_DOTPROD + /* Dotprod computes weighted sums inline, so final reduction is simple */ + s2acc = vaddq_u32(s2acc, s3acc); + pair[0] = vaddvq_u32(adacc); + pair[1] = vaddvq_u32(s2acc); +#else uint16x8x4_t t0_t3 = vld1q_u16_x4_ex(taps, 256); uint16x8x4_t t4_t7 = vld1q_u16_x4_ex(taps + 32, 256); @@ -200,6 +284,7 @@ Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, co s2acc = vaddq_u32(s2acc, s3acc); pair[0] = vaddvq_u32(adacc); pair[1] = vaddvq_u32(s2acc); +#endif pair[0] %= BASE; pair[1] %= BASE; diff --git a/arch/arm/arm_features.c b/arch/arm/arm_features.c index 8f179526e..60ff0220b 100644 --- a/arch/arm/arm_features.c +++ b/arch/arm/arm_features.c @@ -174,6 +174,48 @@ static int arm_has_eor3(void) { return has_eor3; } +static int arm_has_dotprod(void) { + int has_dotprod = 0; +#if defined(__ARM_FEATURE_DOTPROD) + /* Compile-time check */ + has_dotprod = 1; +#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H) +# ifdef HWCAP_ASIMDDP + has_dotprod = (getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0; +# endif +#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H) +# ifdef HWCAP_ASIMDDP + unsigned long hwcap = 0; + elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)); + has_dotprod = (hwcap & HWCAP_ASIMDDP) != 0; +# endif +#elif defined(__FreeBSD__) && defined(ARCH_64BIT) +# ifdef ID_AA64ISAR0_DP_VAL + has_dotprod = getenv("QEMU_EMULATING") == NULL + && ID_AA64ISAR0_DP_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_DP_IMPL; +# endif +#elif defined(__OpenBSD__) && defined(ARCH_64BIT) +# ifdef ID_AA64ISAR0_DP + int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 }; + uint64_t isar0 = 0; + size_t len = sizeof(isar0); + if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) { + has_dotprod = ID_AA64ISAR0_DP(isar0) >= ID_AA64ISAR0_DP_IMPL; + } +# endif +#elif defined(__APPLE__) + int has_feat = 0; + size_t size = sizeof(has_feat); + has_dotprod = sysctlbyname("hw.optional.arm.FEAT_DotProd", &has_feat, &size, NULL, 0) == 0 + && has_feat == 1; +#elif defined(_WIN32) +# ifdef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE + has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE); +# endif +#endif + return has_dotprod; +} + /* AArch64 has neon. */ #ifdef ARCH_32BIT static inline int arm_has_neon(void) { @@ -329,6 +371,7 @@ void Z_INTERNAL arm_check_features(struct arm_cpu_features *features) { features->has_pmull = arm_has_pmull(); features->has_eor3 = arm_has_eor3(); features->has_fast_pmull = features->has_pmull && arm_cpu_has_fast_pmull(); + features->has_dotprod = arm_has_dotprod(); } #endif diff --git a/arch/arm/arm_features.h b/arch/arm/arm_features.h index 2f17a9ddf..28617130a 100644 --- a/arch/arm/arm_features.h +++ b/arch/arm/arm_features.h @@ -9,6 +9,7 @@ struct arm_cpu_features { int has_simd; int has_neon; int has_crc32; + int has_dotprod; int has_pmull; int has_eor3; int has_fast_pmull; diff --git a/arch/arm/arm_functions.h b/arch/arm/arm_functions.h index f2975fadf..6eb72a3da 100644 --- a/arch/arm/arm_functions.h +++ b/arch/arm/arm_functions.h @@ -18,6 +18,11 @@ uint32_t longest_match_roll_neon(deflate_state *const s, uint32_t cur_match); void slide_hash_neon(deflate_state *s); #endif +#ifdef ARM_NEON_DOTPROD +uint32_t adler32_neon_dotprod(uint32_t adler, const uint8_t *buf, size_t len); +uint32_t adler32_copy_neon_dotprod(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len); +#endif + #ifndef ARM_NEON_NATIVE # define ADLER32_FALLBACK # define CHUNKSET_FALLBACK @@ -70,6 +75,13 @@ void slide_hash_armv6(deflate_state *s); # undef native_slide_hash # define native_slide_hash slide_hash_neon # endif +// ARM - NEON DotProd +# ifdef ARM_NEON_DOTPROD_NATIVE +# undef native_adler32 +# define native_adler32 adler32_neon_dotprod +# undef native_adler32_copy +# define native_adler32_copy adler32_copy_neon_dotprod +# endif // ARM - CRC32 # ifdef ARM_CRC32_NATIVE # undef native_crc32 diff --git a/arch/arm/arm_natives.h b/arch/arm/arm_natives.h index 311e33e95..7556577d4 100644 --- a/arch/arm/arm_natives.h +++ b/arch/arm/arm_natives.h @@ -16,6 +16,12 @@ # define ARM_NEON_NATIVE # endif #endif +/* DotProd is optional in ARMv8.2+, mandatory in ARMv8.4+ */ +#if defined(__ARM_FEATURE_DOTPROD) +# ifdef ARM_NEON_DOTPROD +# define ARM_NEON_DOTPROD_NATIVE +# endif +#endif /* CRC32 is optional in ARMv8.0, mandatory in ARMv8.1+ */ #if defined(__ARM_FEATURE_CRC32) || (defined(__ARM_ARCH) && __ARM_ARCH >= 801) # ifdef ARM_CRC32 diff --git a/cmake/detect-intrinsics.cmake b/cmake/detect-intrinsics.cmake index 3b9acfa13..9d15ad23c 100644 --- a/cmake/detect-intrinsics.cmake +++ b/cmake/detect-intrinsics.cmake @@ -334,6 +334,30 @@ macro(check_neon_ld4_intrinsics) set(CMAKE_REQUIRED_FLAGS) endmacro() +macro(check_neon_dotprod_compiler_flag) + if(NOT NATIVEFLAG) + if(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang") + set(NEONDOTPRODFLAG "-march=armv8.2-a+dotprod") + endif() + endif() + set(CMAKE_REQUIRED_FLAGS "${NEONDOTPRODFLAG} ${NATIVEFLAG} ${ZNOLTOFLAG}") + check_c_source_compiles( + "#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_ARM64) || defined(_M_ARM64EC)) + # include + #else + # include + #endif + int main(void) { + uint8x16_t a = vdupq_n_u8(1); + uint8x16_t b = vdupq_n_u8(2); + uint32x4_t c = vdupq_n_u32(0); + c = vdotq_u32(c, a, b); + return vgetq_lane_u32(c, 0); + }" + NEON_HAS_DOTPROD) + set(CMAKE_REQUIRED_FLAGS) +endmacro() + macro(check_pclmulqdq_intrinsics) if(NOT NATIVEFLAG) if(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang" OR CMAKE_C_COMPILER_ID MATCHES "IntelLLVM" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC") diff --git a/configure b/configure index 9c9cff56c..14974bf8b 100755 --- a/configure +++ b/configure @@ -129,6 +129,7 @@ lasxflag="-mlasx" armv8flag= pmulleor3flag= neonflag= +neondotprodflag= rvvflag= rvvzbcflag= zbcflag= @@ -1334,6 +1335,31 @@ EOF fi } +check_neon_dotprod_compiler_flag() { + neondotprodflag="-march=armv8.2-a+dotprod" + cat > $test.c << EOF +#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC)) +# include +#else +# include +#endif +int main(void) { + uint8x16_t a = vdupq_n_u8(1); + uint8x16_t b = vdupq_n_u8(2); + uint32x4_t c = vdupq_n_u32(0); + c = vdotq_u32(c, a, b); + return vgetq_lane_u32(c, 0); +} +EOF + if try $CC -c $CFLAGS $neondotprodflag $test.c; then + NEON_HAS_DOTPROD=1 + echo "Check whether compiler supports NEON dotprod intrinsics ... Yes." | tee -a configure.log + else + NEON_HAS_DOTPROD=0 + echo "Check whether compiler supports NEON dotprod intrinsics ... No." | tee -a configure.log + fi +} + check_neon_ld4_intrinsics() { cat > $test.c << EOF #if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC)) @@ -2011,6 +2037,15 @@ EOF ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_neon.o chunkset_neon.o compare256_neon.o slide_hash_neon.o" ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_neon.lo chunkset_neon.lo compare256_neon.lo slide_hash_neon.lo" + + check_neon_dotprod_compiler_flag + + if test $NEON_HAS_DOTPROD -eq 1 && test $ARCH_64BIT -eq 1; then + CFLAGS="${CFLAGS} -DARM_NEON_DOTPROD" + SFLAGS="${SFLAGS} -DARM_NEON_DOTPROD" + ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_neon_dotprod.o" + ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_neon_dotprod.lo" + fi fi fi @@ -2318,6 +2353,7 @@ echo xsaveflag = $xsaveflag >> configure.log echo armv8flag = $armv8flag >> configure.log echo pmulleor3flag = $pmulleor3flag >> configure.log echo neonflag = $neonflag >> configure.log +echo neondotprodflag = $neondotprodflag >> configure.log echo armv6flag = $armv6flag >> configure.log echo lsxflag = $lsxflag >> configure.log echo lasxflag = $lasxflag >> configure.log @@ -2464,6 +2500,7 @@ sed < $SRCDIR/$ARCHDIR/Makefile.in " /^ARMV8FLAG *=/s#=.*#=$armv8flag# /^PMULLEOR3FLAG *=/s#=.*#=$pmulleor3flag# /^NEONFLAG *=/s#=.*#=$neonflag# +/^NEONDOTPRODFLAG *=/s#=.*#=$neondotprodflag# /^ARMV6FLAG *=/s#=.*#=$armv6flag# /^NOLTOFLAG *=/s#=.*#=$noltoflag# /^S390VXFLAG *=/s#=.*#=$s390vxflag# diff --git a/functable.c b/functable.c index 6ad05f32e..cf453e386 100644 --- a/functable.c +++ b/functable.c @@ -267,6 +267,16 @@ static int init_functable(void) { ft.longest_match_roll = &longest_match_roll_neon; ft.slide_hash = &slide_hash_neon; } +#endif + // ARM - NEON DotProd +#ifdef ARM_NEON_DOTPROD +# ifndef ARM_NEON_DOTPROD_NATIVE + if (cf.arm.has_neon && cf.arm.has_dotprod) +# endif + { + ft.adler32 = &adler32_neon_dotprod; + ft.adler32_copy = &adler32_copy_neon_dotprod; + } #endif // ARM - CRC32 #if defined(ARM_CRC32) && !defined(ARM_PMULL_EOR3_NATIVE) diff --git a/test/benchmarks/benchmark_adler32.cc b/test/benchmarks/benchmark_adler32.cc index 8b0a6beef..ce28f41c3 100644 --- a/test/benchmarks/benchmark_adler32.cc +++ b/test/benchmarks/benchmark_adler32.cc @@ -90,6 +90,9 @@ BENCHMARK_ADLER32(native, native_adler32, 1); #ifdef ARM_NEON BENCHMARK_ADLER32(neon, adler32_neon, test_cpu_features.arm.has_neon); #endif +#ifdef ARM_NEON_DOTPROD +BENCHMARK_ADLER32(neon_dotprod, adler32_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod); +#endif #ifdef PPC_VMX BENCHMARK_ADLER32(vmx, adler32_vmx, test_cpu_features.power.has_altivec); diff --git a/test/benchmarks/benchmark_adler32_copy.cc b/test/benchmarks/benchmark_adler32_copy.cc index 16526cd3f..610911f57 100644 --- a/test/benchmarks/benchmark_adler32_copy.cc +++ b/test/benchmarks/benchmark_adler32_copy.cc @@ -143,6 +143,9 @@ BENCHMARK_ADLER32_COPY(native, native_adler32, native_adler32_copy, 1); #ifdef ARM_NEON BENCHMARK_ADLER32_COPY(neon, adler32_neon, adler32_copy_neon, test_cpu_features.arm.has_neon); #endif +#ifdef ARM_NEON_DOTPROD +BENCHMARK_ADLER32_COPY(neon_dotprod, adler32_neon_dotprod, adler32_copy_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod); +#endif #ifdef PPC_VMX BENCHMARK_ADLER32_COPY(vmx, adler32_vmx, adler32_copy_vmx, test_cpu_features.power.has_altivec); diff --git a/test/test_adler32.cc b/test/test_adler32.cc index 91189a4ae..592480175 100644 --- a/test/test_adler32.cc +++ b/test/test_adler32.cc @@ -47,7 +47,11 @@ TEST_ADLER32(native, native_adler32, 1) #ifdef ARM_NEON TEST_ADLER32(neon, adler32_neon, test_cpu_features.arm.has_neon) -#elif defined(POWER8_VSX) +#endif +#ifdef ARM_NEON_DOTPROD +TEST_ADLER32(neon_dotprod, adler32_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod) +#endif +#if defined(POWER8_VSX) TEST_ADLER32(power8, adler32_power8, test_cpu_features.power.has_arch_2_07) #elif defined(PPC_VMX) TEST_ADLER32(vmx, adler32_vmx, test_cpu_features.power.has_altivec) diff --git a/test/test_adler32_copy.cc b/test/test_adler32_copy.cc index b7ca376b7..b5cab369f 100644 --- a/test/test_adler32_copy.cc +++ b/test/test_adler32_copy.cc @@ -52,7 +52,11 @@ TEST_ADLER32_COPY(c, adler32_copy_c, 1) #ifdef ARM_NEON TEST_ADLER32_COPY(neon, adler32_copy_neon, test_cpu_features.arm.has_neon) -#elif defined(POWER8_VSX) +#endif +#ifdef ARM_NEON_DOTPROD +TEST_ADLER32_COPY(neon_dotprod, adler32_copy_neon_dotprod, test_cpu_features.arm.has_neon && test_cpu_features.arm.has_dotprod) +#endif +#if defined(POWER8_VSX) TEST_ADLER32_COPY(power8, adler32_copy_power8, test_cpu_features.power.has_arch_2_07) #elif defined(PPC_VMX) TEST_ADLER32_COPY(vmx, adler32_copy_vmx, test_cpu_features.power.has_altivec)