/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2020-2021, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*/
+#ifdef HAVE_SVE
+
+/*
+ * blockSingleMask takes in a character set (as masks) and a string and return for each character
+ * of the string weither or not it is part of the set.
+ *
+ * 'shuf_mask_lo_highclear' and 'shuf_mask_lo_highset' are 128-bit masks where each bit
+ * represents whether or not a character is in the character set. The 'highclear' and
+ * 'highset' in the name refers to the MSb of the byte of the character (allowing two
+ * 128-bit masks to cover all 256 values).
+ *
+ * The masks are arrays of 16 bytes each and are encoded this way:
+ * Let C be a character in the set. The bit describing that character is at byte[C%16] and
+ * within that byte, it's at bit[C/16]
+ * As example, 'a' = 0x61, so the resulting mask will be: 0x00 0x40 0x00 0x00 0x00 ...
+ *
+ * Assume both mask are 128b wide. If they are larger, the additional bits must be zero
+ */
+static really_inline
+svuint8_t blockSingleMask(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars) {
+
+ const svuint8_t highconst = svdup_u8(0x80);
+ const svuint8_t pshub_mask = svdup_u8(0x8f);
+ const svuint8_t unique_bit_per_lane_mask = svreinterpret_u8(svdup_u64(0x8040201008040201));
+
+ /*
+ * svtbl does a table lookup. Each byte in the second argument indexes into the array of bytes
+ * in shuf_mask_lo_highclear and saves the result in the corresponding byte of byte_select_low.
+ * We mask the chars so that we are using the low nibble of char as the index but we keep the
+ * MSb so that high characters (not represented by the highclear mask) become an index out of
+ * bounds and result in a 0.
+ */
+ svuint8_t byte_select_low = svtbl(shuf_mask_lo_highclear, svand_x(svptrue_b8(), chars, pshub_mask));
+
+ /*
+ * We flip the MSb of the chars and do the same table lookup with the highset mask.
+ * This way it's the characters with MSb cleared that will result in out of bands indexes.
+ * This allows us to cover the full range (0-127 and 128-255)
+ */
+ svuint8_t char_high_flipped = sveor_x(svptrue_b8(), chars, highconst);
+ svuint8_t byte_select_high = svtbl(shuf_mask_lo_highset, svand_x(svptrue_b8(), char_high_flipped, pshub_mask));
+
+ /*
+ * We now have selected the byte that contain the bit corresponding to the char. We need to
+ * further filter it, otherwise we'd get a match for any character % 16 to a searched character
+ *
+ * The low nibble was used previously to select the byte out of the mask. The high nibble is
+ * used to select the bit out of the byte. So we shift everything right by 4.
+ *
+ * Using svtbl, we can make an array where each element is a different bit. Using the high
+ * nibble we can get a mask selecting only the bit out of a byte that may have the relevant
+ * charset char.
+ */
+ svuint8_t char_high_nibble = svlsr_x(svptrue_b8(), chars, 4);
+ svuint8_t bit_select = svtbl(unique_bit_per_lane_mask, char_high_nibble);
+ /*
+ * For every lane, only one of the byte selected may have a value, so we can OR them. We
+ * then apply the bit_select mask. What is left is the bit in the charset encoding the
+ * character in char. A non zero value means the char was in the charset
+ *
+ * The _x suffix only works if we process a full char vector. If we were to use a partial
+ * vector, then _z and a mask would be required on this svand only. Otherwise, the disabled
+ * lanes may have arbitrary values
+ */
+ svuint8_t res = svand_x(svptrue_b8(), svorr_x(svptrue_b8(), byte_select_low, byte_select_high), bit_select);
+
+ return res;
+}
+#else
+
template <uint16_t S>
static really_inline
const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars) {
return !res.eq(SuperVector<S>::Zeroes());
}
+#endif //HAVE_SVE
\ No newline at end of file
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2020-2023, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
#include "util/supervector/supervector.hpp"
#include "util/match.hpp"
+#ifdef HAVE_SVE
+static really_inline
+svuint8_t blockSingleMask(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars);
+#else
template <uint16_t S>
static really_inline
const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars);
+#endif //HAVE_SVE
#if defined(VS_SIMDE_BACKEND)
#include "x86/truffle.hpp"
#endif
#endif
+#ifdef HAVE_SVE
+
+const u8 *truffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset,
+ const u8 *buf, const u8 *buf_end);
+
+const u8 *rtruffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset,
+ const u8 *buf, const u8 *buf_end);
+
+static really_inline
+const u8 *scanBlock(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars, const u8 *buf, bool forward) {
+
+ const size_t vector_size_int_8 = svcntb();
+
+ const svuint8_t result_mask = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
+ uint64_t index;
+ if (forward) {
+ index = first_non_zero(vector_size_int_8, result_mask);
+ } else {
+ index = last_non_zero(vector_size_int_8, result_mask);
+ }
+
+ if(index < vector_size_int_8) {
+ return buf+index;
+ } else {
+ return NULL;
+ }
+}
+
+really_inline
+const u8 *truffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end) {
+ const int vect_size_int8 = svcntb();
+ // Activate only 16 lanes to read the m128 buffers
+ const svbool_t lane_pred_16 = svwhilelt_b8(0, 16);
+ assert(buf && buf_end);
+ assert(buf < buf_end);
+ DEBUG_PRINTF("truffle %p len %zu\n", buf, buf_end - buf);
+ DEBUG_PRINTF("b %s\n", buf);
+
+ svuint8_t wide_shuf_mask_lo_highclear = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highclear);
+ svuint8_t wide_shuf_mask_lo_highset = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highset);
+
+ const u8 *work_buffer = buf;
+ const u8 *ret_val;
+
+ DEBUG_PRINTF("start %p end %p \n", work_buffer, buf_end);
+ assert(work_buffer < buf_end);
+
+ __builtin_prefetch(work_buffer + 16*64);
+
+ if (work_buffer + vect_size_int8 <= buf_end) {
+ // Reach vector aligned boundaries
+ DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(work_buffer, vect_size_int8));
+ if (!ISALIGNED_N(work_buffer, vect_size_int8)) {
+ svuint8_t chars = svld1(svptrue_b8(), work_buffer);
+ const u8 *alligned_buffer = ROUNDUP_PTR(work_buffer, vect_size_int8);
+ ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer, true);
+ if (ret_val && ret_val < alligned_buffer) return ret_val;
+ work_buffer = alligned_buffer;
+ }
+
+ while(work_buffer + vect_size_int8 <= buf_end) {
+ __builtin_prefetch(work_buffer + 16*64);
+ DEBUG_PRINTF("work_buffer %p \n", work_buffer);
+ svuint8_t chars = svld1(svptrue_b8(), work_buffer);
+ ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer, true);
+ if (ret_val) return ret_val;
+ work_buffer += vect_size_int8;
+ }
+ }
+
+ DEBUG_PRINTF("work_buffer %p e %p \n", work_buffer, buf_end);
+ // finish off tail
+
+ if (work_buffer != buf_end) {
+ svuint8_t chars;
+ const u8* end_buf;
+ if (buf_end - buf < vect_size_int8) {
+ const svbool_t remaining_lanes = svwhilelt_b8(0ll, buf_end - buf);
+ chars = svld1(remaining_lanes, buf);
+ end_buf = buf;
+ } else {
+ chars = svld1(svptrue_b8(), buf_end - vect_size_int8);
+ end_buf = buf_end - vect_size_int8;
+ }
+ ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, end_buf, true);
+ DEBUG_PRINTF("ret_val %p \n", ret_val);
+ if (ret_val && ret_val < buf_end) return ret_val;
+ }
+
+ return buf_end;
+}
+
+really_inline
+const u8 *rtruffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end){
+ const int vect_size_int8 = svcntb();
+ // Activate only 16 lanes to read the m128 buffers
+ const svbool_t lane_pred_16 = svwhilelt_b8(0, 16);
+ assert(buf && buf_end);
+ assert(buf < buf_end);
+ DEBUG_PRINTF("truffle %p len %zu\n", buf, buf_end - buf);
+ DEBUG_PRINTF("b %s\n", buf);
+
+ svuint8_t wide_shuf_mask_lo_highclear = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highclear);
+ svuint8_t wide_shuf_mask_lo_highset = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highset);
+
+ const u8 *work_buffer = buf_end;
+ const u8 *ret_val;
+
+ DEBUG_PRINTF("start %p end %p \n", buf, work_buffer);
+ assert(work_buffer > buf);
+
+ __builtin_prefetch(work_buffer - 16*64);
+
+ if (work_buffer - vect_size_int8 >= buf) {
+ // Reach vector aligned boundaries
+ DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(work_buffer, vect_size_int8));
+ if (!ISALIGNED_N(work_buffer, vect_size_int8)) {
+ svuint8_t chars = svld1(svptrue_b8(), work_buffer - vect_size_int8);
+ const u8 *alligned_buffer = ROUNDDOWN_PTR(work_buffer, vect_size_int8);
+ ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer - vect_size_int8, false);
+ DEBUG_PRINTF("ret_val %p \n", ret_val);
+ if (ret_val >= alligned_buffer) return ret_val;
+ work_buffer = alligned_buffer;
+ }
+
+ while (work_buffer - vect_size_int8 >= buf) {
+ DEBUG_PRINTF("aligned %p \n", work_buffer);
+ // On large packet buffers, this prefetch appears to get us about 2%.
+ __builtin_prefetch(work_buffer - 16*64);
+
+ work_buffer -= vect_size_int8;
+ svuint8_t chars = svld1(svptrue_b8(), work_buffer);
+ ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer, false);
+ if (ret_val) return ret_val;
+ }
+ }
+
+ DEBUG_PRINTF("tail work_buffer %p e %p \n", buf, work_buffer);
+ // finish off head
+
+ if (work_buffer != buf) {
+ svuint8_t chars;
+ if (buf_end - buf < vect_size_int8) {
+ const svbool_t remaining_lanes = svwhilele_b8(0ll, buf_end - buf);
+ chars = svld1(remaining_lanes, buf);
+ } else {
+ chars = svld1(svptrue_b8(), buf);
+ }
+ ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, buf, false);
+ DEBUG_PRINTF("ret_val %p \n", ret_val);
+ if (ret_val && ret_val < buf_end) return ret_val;
+ }
+
+ return buf - 1;
+}
+#else
template <uint16_t S>
static really_inline
const u8 *fwdBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars, const u8 *buf) {
const u8 *d = buf;
const u8 *rv;
- DEBUG_PRINTF("start %p end %p \n", d, buf_end);
- assert(d < buf_end);
-
- __builtin_prefetch(d + 64);
- __builtin_prefetch(d + 2*64);
- __builtin_prefetch(d + 3*64);
- __builtin_prefetch(d + 4*64);
+ __builtin_prefetch(d + 16*64);
DEBUG_PRINTF("start %p end %p \n", d, buf_end);
assert(d < buf_end);
if (d + S <= buf_end) {
}
while(d + S <= buf_end) {
- __builtin_prefetch(d + 64);
+ __builtin_prefetch(d + 16*64);
DEBUG_PRINTF("d %p \n", d);
SuperVector<S> chars = SuperVector<S>::load(d);
rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
const u8 *d = buf_end;
const u8 *rv;
- __builtin_prefetch(d - 64);
- __builtin_prefetch(d - 2*64);
- __builtin_prefetch(d - 3*64);
- __builtin_prefetch(d - 4*64);
+ __builtin_prefetch(d - 16*64);
DEBUG_PRINTF("start %p end %p \n", buf, d);
assert(d > buf);
if (d - S >= buf) {
while (d - S >= buf) {
DEBUG_PRINTF("aligned %p \n", d);
// On large packet buffers, this prefetch appears to get us about 2%.
- __builtin_prefetch(d - 64);
+ __builtin_prefetch(d - 16*64);
d -= S;
SuperVector<S> chars = SuperVector<S>::load(d);
return buf - 1;
}
+#endif //HAVE_SVE
\ No newline at end of file
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2020-2021, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
}
}
+#ifdef HAVE_SVE
+
+
+static really_inline
+uint64_t last_non_zero_real(svuint8_t mask) {
+ const svuint64_t leading_zeros = svclz_x(svptrue_b64(), svreinterpret_u64(mask));
+
+ uint64_t last_active_lane;
+
+ svbool_t remaining_mask = svptrue_b64();
+ uint64_t i = 0;
+ while(svptest_any(svptrue_b64(), remaining_mask)) {
+ svbool_t single_lane_mask = svpnext_b64(remaining_mask, svpfalse());
+ remaining_mask = sveor_z(svptrue_b64(), remaining_mask, single_lane_mask);
+ uint64_t active_element = svlastb(single_lane_mask, leading_zeros);
+ if(active_element<64) {
+ uint64_t lane_index = (i+1)*8 - (active_element/8) - 1;
+ last_active_lane = lane_index;
+ }
+ i++;
+ }
+ return last_active_lane;
+}
+
+/*
+ * It is assumed mask have the value 0 for all inactive lanes, if any.
+ */
+static really_inline
+uint64_t last_non_zero(const size_t vector_size_int_8, svuint8_t mask) {
+ const svbool_t result_pred = svcmpne(svptrue_b8(), mask, 0);
+
+ if (svptest_any(svptrue_b8(), result_pred)) {
+ return last_non_zero_real(mask);
+ } else {
+ return vector_size_int_8;
+ }
+}
+
+/*
+ * It is assumed mask have the value 0 for all inactive lanes, if any.
+ */
+static really_inline
+uint64_t first_non_zero(const size_t vector_size_int_8, svuint8_t mask) {
+ const svbool_t result_pred = svcmpne(svptrue_b8(), mask, 0);
+
+ if (svptest_any(svptrue_b8(), result_pred)) {
+
+ // We don't have a CTZ instruction but we can work around by reversing the lane order
+ const svuint64_t rev_large_res = svreinterpret_u64(svrev(mask));
+ // Now each pack of 8 leading 0 means one empty lane. So if we have 18 leading 0,
+ // that means the third lane have a matching character.
+ uint64_t first_active_lane = last_non_zero_real(svreinterpret_u8(rev_large_res));
+ // We reversed the lanes, so we reverse back the index
+ first_active_lane = (vector_size_int_8-1) - first_active_lane;
+ return first_active_lane;
+ } else {
+ return vector_size_int_8;
+ }
+}
+
+#endif //HAVE_SVE
\ No newline at end of file
projects / thirdparty / vectorscan.git / commitdiff
